diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..5e267373d8486a41199329d38149bb5db7fb4e8b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,64 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_1_2025-01-18-21-04-56.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_2_2025-01-18-21-05-37.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_16_2025-01-18-21-17-07.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/run-2t6elatx.wandb filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_17_2025-01-18-21-18-04.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/13_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/18_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/15_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/16_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/17_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/eval_video/14_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_15_2025-01-18-21-16-11.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_13_2025-01-18-21-14-07.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_11_2025-01-18-21-12-26.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_4_2025-01-18-21-07-10.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_10_2025-01-18-21-11-40.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_9_2025-01-18-21-10-52.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_12_2025-01-18-21-13-08.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_14_2025-01-18-21-15-17.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_18_2025-01-18-21-19-11.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_6_2025-01-18-21-08-37.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_19_2025-01-18-21-20-12.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_0_2025-01-18-21-04-13.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_5_2025-01-18-21-07-55.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_3_2025-01-18-21-06-23.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_8_2025-01-18-21-10-08.bag filter=lfs diff=lfs merge=lfs -text +2t6elatx/episode_rosbags/episode_7_2025-01-18-21-09-25.bag filter=lfs diff=lfs merge=lfs -text diff --git a/2t6elatx/.hydra/config.yaml b/2t6elatx/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e205199f663cb5a4be0ac1c8b8e854ceb56738d5 --- /dev/null +++ b/2t6elatx/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3465_2' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/2t6elatx/.hydra/hydra.yaml b/2t6elatx/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5d27091e48b796a7e605f993ecf8a3601486ee9 --- /dev/null +++ b/2t6elatx/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/2t6elatx/.hydra/overrides.yaml b/2t6elatx/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/2t6elatx/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/2t6elatx/episode_rosbags/aligned_depth_to_color_K.npy b/2t6elatx/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/2t6elatx/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/2t6elatx/episode_rosbags/cam_tf_world.npy b/2t6elatx/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/2t6elatx/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/2t6elatx/episode_rosbags/color_K.npy b/2t6elatx/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/2t6elatx/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/2t6elatx/episode_rosbags/depth_K.npy b/2t6elatx/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/2t6elatx/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/2t6elatx/episode_rosbags/episode_0_2025-01-18-21-04-13.bag b/2t6elatx/episode_rosbags/episode_0_2025-01-18-21-04-13.bag new file mode 100644 index 0000000000000000000000000000000000000000..269d3b4a73a814fb5c8e73b145bfa505cc493fd9 --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_0_2025-01-18-21-04-13.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1872349953eccf8849a1fb9fd5e5f1241fce7d7baea936ada3dd1653cf0d5e91 +size 1407175014 diff --git a/2t6elatx/episode_rosbags/episode_10_2025-01-18-21-11-40.bag b/2t6elatx/episode_rosbags/episode_10_2025-01-18-21-11-40.bag new file mode 100644 index 0000000000000000000000000000000000000000..b289a7a98e3b758a33e2196303b40d4a50a31878 --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_10_2025-01-18-21-11-40.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b599e7ccbf1302b15e35e2f1931a937e511c6e701263bf7be0fa797fe0ae3a0d +size 1409127344 diff --git a/2t6elatx/episode_rosbags/episode_11_2025-01-18-21-12-26.bag b/2t6elatx/episode_rosbags/episode_11_2025-01-18-21-12-26.bag new file mode 100644 index 0000000000000000000000000000000000000000..3f389ec8ac49d281901973faf2b270a41cb73591 --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_11_2025-01-18-21-12-26.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fc10afd44a7182f78672111482f66e37fbbe5a90cf9556ba54ac881bbe32e27 +size 1007051003 diff --git a/2t6elatx/episode_rosbags/episode_12_2025-01-18-21-13-08.bag b/2t6elatx/episode_rosbags/episode_12_2025-01-18-21-13-08.bag new file mode 100644 index 0000000000000000000000000000000000000000..135a80223f004bd35392619f6dd8fd8b72b53c6a --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_12_2025-01-18-21-13-08.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:257f80c6b7af66ad5da618d361a32d45361769dfdd90b943323dedd71712f5f7 +size 1409931840 diff --git a/2t6elatx/episode_rosbags/episode_13_2025-01-18-21-14-07.bag b/2t6elatx/episode_rosbags/episode_13_2025-01-18-21-14-07.bag new file mode 100644 index 0000000000000000000000000000000000000000..5e08d47a8e03effbd909e97c4cc906ddf164a178 --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_13_2025-01-18-21-14-07.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84ed99664ae4b60c5829bbc025d5b59d11d47b5474f152951315eb05072c918d +size 1415509593 diff --git a/2t6elatx/episode_rosbags/episode_14_2025-01-18-21-15-17.bag b/2t6elatx/episode_rosbags/episode_14_2025-01-18-21-15-17.bag new file mode 100644 index 0000000000000000000000000000000000000000..27a989cc5141f45cff961dfc64400166e4ea9934 --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_14_2025-01-18-21-15-17.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:961568dad041077687218a48ea794e048d5f7a12eeb57b90a51ceda472e15f17 +size 1414426233 diff --git a/2t6elatx/episode_rosbags/episode_15_2025-01-18-21-16-11.bag b/2t6elatx/episode_rosbags/episode_15_2025-01-18-21-16-11.bag new file mode 100644 index 0000000000000000000000000000000000000000..a18b31c9f829fc59899e4a15ceec317bde9c2e61 --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_15_2025-01-18-21-16-11.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fef6fb8feb6aa15f0e3bda64276d811edd8bef5a0153d444f816b8040314ac40 +size 1414855271 diff --git a/2t6elatx/episode_rosbags/episode_16_2025-01-18-21-17-07.bag b/2t6elatx/episode_rosbags/episode_16_2025-01-18-21-17-07.bag new file mode 100644 index 0000000000000000000000000000000000000000..548aa2da3aeb4eb2062e5831410372a508235f19 --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_16_2025-01-18-21-17-07.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8817b81cf612d786a57aecc1c59417d08d49c07463a98dc5bda5f7602ccae496 +size 1412731823 diff --git a/2t6elatx/episode_rosbags/episode_17_2025-01-18-21-18-04.bag b/2t6elatx/episode_rosbags/episode_17_2025-01-18-21-18-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..3356922000cf1087fcc2327c31f11743499be298 --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_17_2025-01-18-21-18-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f88c4b18831a901ce152c28abf7f83b5958bdcdd94d1387073c14203ae3be5d9 +size 1409566815 diff --git a/2t6elatx/episode_rosbags/episode_18_2025-01-18-21-19-11.bag b/2t6elatx/episode_rosbags/episode_18_2025-01-18-21-19-11.bag new file mode 100644 index 0000000000000000000000000000000000000000..f3df68597f6b810ffd870820b9d9593742c07836 --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_18_2025-01-18-21-19-11.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07eb7a16315128141c9c3beae0498f8e938715b715fee2a52e82163ad2f833dc +size 1417757518 diff --git a/2t6elatx/episode_rosbags/episode_19_2025-01-18-21-20-12.bag b/2t6elatx/episode_rosbags/episode_19_2025-01-18-21-20-12.bag new file mode 100644 index 0000000000000000000000000000000000000000..9ab24b396a13be3ceb8938a0d1e25ccf21c07af2 --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_19_2025-01-18-21-20-12.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69dfb9875fda29b8e35a4dba7d003760d956a15a7b4132093eaac27f9636748a +size 1413615431 diff --git a/2t6elatx/episode_rosbags/episode_1_2025-01-18-21-04-56.bag b/2t6elatx/episode_rosbags/episode_1_2025-01-18-21-04-56.bag new file mode 100644 index 0000000000000000000000000000000000000000..62bbd1ee4d3f28f017cb6294a5bf4ea822588803 --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_1_2025-01-18-21-04-56.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbd9acf3add2db6187208b9032897265d3966b6d6a2ce3e5b5d59d2f90f9b2c6 +size 1412596618 diff --git a/2t6elatx/episode_rosbags/episode_2_2025-01-18-21-05-37.bag b/2t6elatx/episode_rosbags/episode_2_2025-01-18-21-05-37.bag new file mode 100644 index 0000000000000000000000000000000000000000..a4b64f5b7aa5f3824c9b8943e9fb17d7eba5be48 --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_2_2025-01-18-21-05-37.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:019e049f738a93b244e88c9eae7d66d6113d658f51ae85cb73bb8255dac02dfc +size 1414479183 diff --git a/2t6elatx/episode_rosbags/episode_3_2025-01-18-21-06-23.bag b/2t6elatx/episode_rosbags/episode_3_2025-01-18-21-06-23.bag new file mode 100644 index 0000000000000000000000000000000000000000..bbd8612e02cd8bc0cd5b64dc63f7d9cd657be07c --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_3_2025-01-18-21-06-23.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:175c64bbdb432de66031b8ea81648baf29e5c7d2d399bbbaca8a717a1436ec50 +size 1412444152 diff --git a/2t6elatx/episode_rosbags/episode_4_2025-01-18-21-07-10.bag b/2t6elatx/episode_rosbags/episode_4_2025-01-18-21-07-10.bag new file mode 100644 index 0000000000000000000000000000000000000000..0dba090d680fdbe6ceb38bb12f14d66655062f76 --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_4_2025-01-18-21-07-10.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cf75204be009154a9b22ec9560026ea160a25e431df314747f0ab6b50a8d3cf +size 1412858742 diff --git a/2t6elatx/episode_rosbags/episode_5_2025-01-18-21-07-55.bag b/2t6elatx/episode_rosbags/episode_5_2025-01-18-21-07-55.bag new file mode 100644 index 0000000000000000000000000000000000000000..233a34bcf7fb613e3a2a11752260165f98e7431f --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_5_2025-01-18-21-07-55.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74da96cc4d468402255d4a21e4b5e5eb9d509ab63e4b82a723b74cf9c80753b1 +size 1417869361 diff --git a/2t6elatx/episode_rosbags/episode_6_2025-01-18-21-08-37.bag b/2t6elatx/episode_rosbags/episode_6_2025-01-18-21-08-37.bag new file mode 100644 index 0000000000000000000000000000000000000000..1db97629993874999d0f55f117376265fe71b15d --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_6_2025-01-18-21-08-37.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03c5783f163f299f6f0c8d9ff8b8a37f764266ebad466282cfacee7e89c4121e +size 1411005878 diff --git a/2t6elatx/episode_rosbags/episode_7_2025-01-18-21-09-25.bag b/2t6elatx/episode_rosbags/episode_7_2025-01-18-21-09-25.bag new file mode 100644 index 0000000000000000000000000000000000000000..2013ff58ea452d9cbc1532afe59a5bd24c4faa80 --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_7_2025-01-18-21-09-25.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1195621f8cc28aea2904dc45a18a032f93ea0d08cc6b93d93542c2960f9a4e15 +size 1410621681 diff --git a/2t6elatx/episode_rosbags/episode_8_2025-01-18-21-10-08.bag b/2t6elatx/episode_rosbags/episode_8_2025-01-18-21-10-08.bag new file mode 100644 index 0000000000000000000000000000000000000000..174c2cbdc930503f2adf368757af1a0efa00b662 --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_8_2025-01-18-21-10-08.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e932d40f0f50fb7b8050b4f3476f405d7ef0da75a31c6d64613392ea5819650 +size 1406130608 diff --git a/2t6elatx/episode_rosbags/episode_9_2025-01-18-21-10-52.bag b/2t6elatx/episode_rosbags/episode_9_2025-01-18-21-10-52.bag new file mode 100644 index 0000000000000000000000000000000000000000..741378d06370d8af3f17b7a23a49d56025bf790f --- /dev/null +++ b/2t6elatx/episode_rosbags/episode_9_2025-01-18-21-10-52.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30ce21c4aa3f135bcb244a87ed9fde90881dce5ba3d3a628b92f6f43d3e964b5 +size 1412543629 diff --git a/2t6elatx/eval_robot.log b/2t6elatx/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..901e5035465993450e0a1a3c132ae359ebea4c38 --- /dev/null +++ b/2t6elatx/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-18 21:03:38,176][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-18 21:03:38,179][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/2t6elatx/eval_video/0_eval.mp4 b/2t6elatx/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3b2695fe0c174de459d9baef2ad0da8a7c4fdf32 --- /dev/null +++ b/2t6elatx/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:373935c9a53d50a21516ae9864a2b682a53224d448bd3a5444b7c73d070560df +size 1151995 diff --git a/2t6elatx/eval_video/10_eval.mp4 b/2t6elatx/eval_video/10_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0cdb714b9d214b4f677523a9ead2078bb742eb8e --- /dev/null +++ b/2t6elatx/eval_video/10_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:194dcd6049c4233da56c1320d7fff1f54a4b9ebb30a13b10b4abae786a908461 +size 1191627 diff --git a/2t6elatx/eval_video/11_eval.mp4 b/2t6elatx/eval_video/11_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9a195cbb4d8ef55cb90fd9c89d86c2794d7a82a5 --- /dev/null +++ b/2t6elatx/eval_video/11_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:136a95e750557f39dd091db067809586d46c1938c60d2ff38e1ecee849529ad5 +size 1260306 diff --git a/2t6elatx/eval_video/12_eval.mp4 b/2t6elatx/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cb7c6fccbb088173b7ddfe494cfafca379001149 --- /dev/null +++ b/2t6elatx/eval_video/12_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a945bf8da73837c50240f66d862b85dfefc2c9ef4223dc5b6af7da13b3cc2a7 +size 1507207 diff --git a/2t6elatx/eval_video/13_eval.mp4 b/2t6elatx/eval_video/13_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6251cf91ea54958b49ae4f6c13a89b070b1721ab --- /dev/null +++ b/2t6elatx/eval_video/13_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f717f47f7f63a8b7782024933ba07b6251ff2c1cd576b134150c6f7e03fa672b +size 1554780 diff --git a/2t6elatx/eval_video/14_eval.mp4 b/2t6elatx/eval_video/14_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e109051b62c8c22ff7a94e1d0a12321222e46827 --- /dev/null +++ b/2t6elatx/eval_video/14_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cb31531142c32910aad715f3f80bc83b51b20e8f8ba2350371fc71e36b656c4 +size 1617287 diff --git a/2t6elatx/eval_video/15_eval.mp4 b/2t6elatx/eval_video/15_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67098eefdbe36eae0c3f5392164a79e4dc8e2f0a --- /dev/null +++ b/2t6elatx/eval_video/15_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7379e8bc33b5c38d196c92d0056b519a77a0a35feba4aafdac832c3cd26ade2e +size 1465950 diff --git a/2t6elatx/eval_video/16_eval.mp4 b/2t6elatx/eval_video/16_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2f646bb84f96fde702308f149b8ae65155c625cc --- /dev/null +++ b/2t6elatx/eval_video/16_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dec0060310bb185ed9e7be5a700a91ea3a1257e2f1d26935faecf75e44c0871e +size 1506659 diff --git a/2t6elatx/eval_video/17_eval.mp4 b/2t6elatx/eval_video/17_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95d49c5f9ad469e3e0344ffd56e3a2b291399ff8 --- /dev/null +++ b/2t6elatx/eval_video/17_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb16d97a7d17e520d767ebbcee4818f5adbf7a703c8ec5531ce2779703c79eb1 +size 1705295 diff --git a/2t6elatx/eval_video/18_eval.mp4 b/2t6elatx/eval_video/18_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c858cbdbdd8f94d5b3b89b922f18b67ec01e034c --- /dev/null +++ b/2t6elatx/eval_video/18_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c883e9e40544a093c095acd47f766267da343a815cbe528c6506ef129772088e +size 1688371 diff --git a/2t6elatx/eval_video/19_eval.mp4 b/2t6elatx/eval_video/19_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..46ea635bfb7944c481e0262ec56febffcab0ae7c --- /dev/null +++ b/2t6elatx/eval_video/19_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ae4f8c9bd51774c0b8924187da80f36298b869aeebdb7907814ff220087833a +size 1632571 diff --git a/2t6elatx/eval_video/1_eval.mp4 b/2t6elatx/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d65eb7187572828c36ec169e4e63a4ff62df3ab0 --- /dev/null +++ b/2t6elatx/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f136ddfa6a75f8c38319eb69e397cb0cf05992c189494f06c72a123d44e57cad +size 1169127 diff --git a/2t6elatx/eval_video/2_eval.mp4 b/2t6elatx/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ee8814db7459a2c5f5c65b9d40795ff8ad61113d --- /dev/null +++ b/2t6elatx/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9019146f9223df893e271a59c4c8538311ba44dc2fd24a72c9dd4b2c64812680 +size 1170383 diff --git a/2t6elatx/eval_video/3_eval.mp4 b/2t6elatx/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9bb033310d0963d3d393c813ecc100f378b79c9d --- /dev/null +++ b/2t6elatx/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9352300f4a1f2d4fc59db4120a6eeb7a174fd0b71713a6348112847f63b66a3 +size 1198800 diff --git a/2t6elatx/eval_video/4_eval.mp4 b/2t6elatx/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..37fbd02ec40e4f00d8406cf967974c0c31065d8e --- /dev/null +++ b/2t6elatx/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff051e1686bf761b709a2c0ca3d274928506972c8da93aa7332f58ee411cb53a +size 1236656 diff --git a/2t6elatx/eval_video/5_eval.mp4 b/2t6elatx/eval_video/5_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c757dbfc09bf24ecb7c26691a010e1afb7348b19 --- /dev/null +++ b/2t6elatx/eval_video/5_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a0fa497056b618ea0c00d6fc66b9c6ec244aca7406d79762d07999bb1f21f05 +size 1158233 diff --git a/2t6elatx/eval_video/6_eval.mp4 b/2t6elatx/eval_video/6_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ef0b304c33ea4347ec5779387dd30bd13cc7d0e8 --- /dev/null +++ b/2t6elatx/eval_video/6_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a90b503bf7614a8b51c57e646cac1e6c87e69c6f37fa340d991743bbf1419c98 +size 1334222 diff --git a/2t6elatx/eval_video/7_eval.mp4 b/2t6elatx/eval_video/7_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..87b5a708b3f6220669f8c15571f717c933fdd35f --- /dev/null +++ b/2t6elatx/eval_video/7_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9b2fef5e246e9ba0a8b58badafa48d2ec104d3ee52c60f44f6483987e7d9ea6 +size 1195242 diff --git a/2t6elatx/eval_video/8_eval.mp4 b/2t6elatx/eval_video/8_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0f71357a0c413878b9781be35965a7fe986780e8 --- /dev/null +++ b/2t6elatx/eval_video/8_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4562dd864115606b629194b8fd32ec23a69600d8aa0c75918b7b283689ef65e7 +size 1360171 diff --git a/2t6elatx/eval_video/9_eval.mp4 b/2t6elatx/eval_video/9_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1f8e5af7b951dfdb2cd84a485a0e1860c144f89e --- /dev/null +++ b/2t6elatx/eval_video/9_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cb54919d1152aace357fcead199b255676220f42ac2229e103ed67f6e84045c +size 1409917 diff --git a/2t6elatx/tb/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 b/2t6elatx/tb/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 new file mode 100644 index 0000000000000000000000000000000000000000..e60a38c18c755022985fa10c5e780e2aed833e33 --- /dev/null +++ b/2t6elatx/tb/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51a9fdeb8b70e9cbbe818217e4d4984a344f906dca8e6db5024d9dfb849c37e8 +size 22784 diff --git a/2t6elatx/wandb/debug-internal.log b/2t6elatx/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9db61d47fbd543c4968821d00ad8f3ba31fe4b28 --- /dev/null +++ b/2t6elatx/wandb/debug-internal.log @@ -0,0 +1,2254 @@ +2025-01-18 21:03:44,037 INFO StreamThr :2577265 [internal.py:wandb_internal():85] W&B internal server running at pid: 2577265, started at: 2025-01-18 21:03:44.037545 +2025-01-18 21:03:44,039 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status +2025-01-18 21:03:44,053 INFO WriterThread:2577265 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/run-2t6elatx.wandb +2025-01-18 21:03:44,054 DEBUG SenderThread:2577265 [sender.py:send():379] send: header +2025-01-18 21:03:44,061 DEBUG SenderThread:2577265 [sender.py:send():379] send: run +2025-01-18 21:03:44,485 INFO SenderThread:2577265 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files +2025-01-18 21:03:44,485 INFO SenderThread:2577265 [sender.py:_start_run_threads():1188] run started: 2t6elatx with start time 1737252224.03754 +2025-01-18 21:03:44,500 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: check_version +2025-01-18 21:03:44,500 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: check_version +2025-01-18 21:03:44,548 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: run_start +2025-01-18 21:03:44,553 DEBUG HandlerThread:2577265 [system_info.py:__init__():26] System info init +2025-01-18 21:03:44,553 DEBUG HandlerThread:2577265 [system_info.py:__init__():41] System info init done +2025-01-18 21:03:44,553 INFO HandlerThread:2577265 [system_monitor.py:start():194] Starting system monitor +2025-01-18 21:03:44,554 INFO SystemMonitor:2577265 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-18 21:03:44,554 INFO HandlerThread:2577265 [system_monitor.py:probe():214] Collecting system info +2025-01-18 21:03:44,554 INFO SystemMonitor:2577265 [interfaces.py:start():188] Started cpu monitoring +2025-01-18 21:03:44,555 INFO SystemMonitor:2577265 [interfaces.py:start():188] Started disk monitoring +2025-01-18 21:03:44,555 INFO SystemMonitor:2577265 [interfaces.py:start():188] Started gpu monitoring +2025-01-18 21:03:44,556 INFO SystemMonitor:2577265 [interfaces.py:start():188] Started memory monitoring +2025-01-18 21:03:44,557 INFO SystemMonitor:2577265 [interfaces.py:start():188] Started network monitoring +2025-01-18 21:03:44,567 DEBUG HandlerThread:2577265 [system_info.py:probe():152] Probing system +2025-01-18 21:03:44,568 DEBUG HandlerThread:2577265 [system_info.py:_probe_git():137] Probing git +2025-01-18 21:03:44,573 DEBUG HandlerThread:2577265 [system_info.py:_probe_git():145] Probing git done +2025-01-18 21:03:44,573 DEBUG HandlerThread:2577265 [system_info.py:probe():200] Probing system done +2025-01-18 21:03:44,573 DEBUG HandlerThread:2577265 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-19T02:03:44.567264', 'startedAt': '2025-01-19T02:03:44.029624', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3558.444375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4480.37, 'min': 800.0, 'max': 4700.0}, {'current': 4488.089, 'min': 800.0, 'max': 4700.0}, {'current': 4562.354, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 658.0720520019531}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-18 21:03:44,573 INFO HandlerThread:2577265 [system_monitor.py:probe():224] Finished collecting system info +2025-01-18 21:03:44,573 INFO HandlerThread:2577265 [system_monitor.py:probe():227] Publishing system info +2025-01-18 21:03:44,573 DEBUG HandlerThread:2577265 [system_info.py:_save_code():44] Saving code +2025-01-18 21:03:44,574 DEBUG HandlerThread:2577265 [system_info.py:_save_code():67] Saving code done +2025-01-18 21:03:44,575 DEBUG HandlerThread:2577265 [system_info.py:_save_patches():84] Saving git patches +2025-01-18 21:03:44,669 DEBUG HandlerThread:2577265 [system_info.py:_save_patches():126] Saving git patches done +2025-01-18 21:03:44,670 INFO HandlerThread:2577265 [system_monitor.py:probe():229] Finished publishing system info +2025-01-18 21:03:44,673 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:03:44,673 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-18 21:03:44,673 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-18 21:03:44,673 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-18 21:03:44,674 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-18 21:03:44,904 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: python_packages +2025-01-18 21:03:44,904 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:03:44,904 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: python_packages +2025-01-18 21:03:44,905 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:03:44,906 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:44,906 INFO HandlerThread:2577265 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/tb" + save: true +} +control { + relay_id: "bcf15667ba81445d81be1061d1a1388d" +} +_info { + stream_id: "2t6elatx" +} + +2025-01-18 21:03:44,907 DEBUG HandlerThread:2577265 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-18 21:03:44,931 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/a4oe7r2l-wandb-metadata.json +2025-01-18 21:03:44,938 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/cl0olw9t-code/FISH/eval_robot.py +2025-01-18 21:03:44,946 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/h5brxswo-diff.patch +2025-01-18 21:03:44,949 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/f3we0mxa-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 21:03:45,019 DEBUG SenderThread:2577265 [sender.py:send():379] send: telemetry +2025-01-18 21:03:45,019 DEBUG SenderThread:2577265 [sender.py:send():379] send: tbrecord +2025-01-18 21:03:45,020 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:03:45,020 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 with policy live +2025-01-18 21:03:45,201 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/vu00izql-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:03:45,487 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/requirements.txt +2025-01-18 21:03:45,487 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:03:45,487 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/code/FISH/eval_robot.py +2025-01-18 21:03:45,487 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 21:03:45,487 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/diff.patch +2025-01-18 21:03:45,488 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:03:45,488 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-metadata.json +2025-01-18 21:03:45,488 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/code/FISH +2025-01-18 21:03:45,488 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/code +2025-01-18 21:03:45,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:46,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:47,487 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:03:47,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:48,880 DEBUG SenderThread:2577265 [sender.py:send():379] send: config +2025-01-18 21:03:48,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:49,889 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:03:49,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:50,906 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:51,488 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:03:51,906 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:52,906 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:53,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:54,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:03:54,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:55,489 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:03:55,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:56,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:57,489 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:03:57,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:58,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:59,904 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:03:59,904 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:03:59,945 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:00,046 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:00,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:01,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:02,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:03,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:04,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:05,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:05,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:06,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:07,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:08,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:09,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:10,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:11,493 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:11,637 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:11,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:12,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:13,494 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:13,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:14,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:04:14,905 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:04:14,945 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:15,494 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:15,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:16,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:16,986 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:17,495 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/config.yaml +2025-01-18 21:04:17,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:18,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:19,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:20,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:21,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:22,906 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:22,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:23,911 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:24,913 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:25,913 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:26,913 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:27,906 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:27,913 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:28,913 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:29,904 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:04:29,904 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:04:29,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:30,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:31,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:32,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:33,058 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:33,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:34,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:35,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:36,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:37,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:38,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:38,915 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:39,915 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:40,915 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:41,240 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:04:41,240 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 with policy now +2025-01-18 21:04:41,240 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:04:41,241 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:04:41,241 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:04:41,242 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:04:41,503 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:41,503 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:04:41,503 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 +2025-01-18 21:04:41,503 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media +2025-01-18 21:04:41,503 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:04:41,503 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos +2025-01-18 21:04:41,508 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/a4g7ag7y-media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 +2025-01-18 21:04:41,799 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/9nlf524s-media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 +2025-01-18 21:04:41,915 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:42,915 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:43,503 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:43,915 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:44,252 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:44,557 DEBUG SystemMonitor:2577265 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-18 21:04:44,558 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:04:44,904 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:04:44,905 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:04:44,916 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:45,917 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:46,916 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:47,505 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:47,916 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:48,917 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:49,505 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:49,609 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:49,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:50,917 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:51,917 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:52,937 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:53,507 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:04:53,768 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/i4lnxezk-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:04:53,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:54,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:54,917 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:55,507 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:55,918 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:56,918 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:57,918 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:58,918 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:59,509 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:59,904 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:04:59,905 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:04:59,945 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:00,029 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:00,918 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:01,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:02,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:03,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:04,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:05,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:05,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:06,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:07,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:08,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:09,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:10,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:10,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:11,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:12,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:13,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:14,566 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:05:14,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:05:14,905 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:05:14,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:15,514 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:15,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:15,990 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:16,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:17,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:18,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:19,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:20,922 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:21,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:21,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:22,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:23,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:24,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:25,145 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:05:25,146 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:05:25,146 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 with policy now +2025-01-18 21:05:25,146 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:05:25,146 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:05:25,146 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:05:25,416 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/u5p126k7-media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 +2025-01-18 21:05:25,517 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:25,517 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:05:25,517 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 +2025-01-18 21:05:25,517 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:05:25,822 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/jpfn1j2z-media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 +2025-01-18 21:05:25,926 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:26,925 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:27,151 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:27,518 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:27,926 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:28,927 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:29,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:05:29,910 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:05:29,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:30,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:31,519 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:31,926 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:32,573 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:32,933 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:33,520 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:33,931 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:34,313 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:05:34,313 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:05:34,313 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:05:34,520 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:05:34,520 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:05:34,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:35,521 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:35,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:36,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:37,521 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:37,636 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:37,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:38,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:39,522 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:39,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:40,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:41,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:42,911 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:42,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:43,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:44,559 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:05:44,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:05:44,905 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:05:44,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:45,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:46,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:47,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:47,993 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:48,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:49,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:50,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:51,526 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:51,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:52,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:53,911 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:53,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:54,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:55,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:56,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:57,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:58,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:59,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:05:59,905 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:05:59,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:59,997 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:00,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:01,931 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:02,931 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:03,932 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:04,932 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:05,193 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:05,941 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:06,234 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:06:06,234 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 with policy now +2025-01-18 21:06:06,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:06:06,237 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:06:06,237 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:06:06,238 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:06:06,531 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:06:06,531 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 +2025-01-18 21:06:06,531 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:06:06,538 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/8kjphwxb-media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 +2025-01-18 21:06:06,866 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/c5b0q6f7-media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 +2025-01-18 21:06:06,934 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:07,531 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:07,937 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:08,937 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:09,938 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:10,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:10,940 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:11,532 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:11,938 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:12,939 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:13,533 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:13,943 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:14,559 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:06:14,934 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:06:14,934 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:06:14,977 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:15,534 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:15,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:16,004 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:16,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:17,950 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:18,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:19,535 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:06:19,828 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/5sqph8os-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:06:19,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:20,151 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:06:20,151 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:06:20,151 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:06:20,536 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:06:20,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:21,536 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:21,557 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:21,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:22,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:23,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:24,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:25,537 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:25,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:26,922 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:26,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:27,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:28,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:29,539 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:29,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:06:29,920 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:06:29,950 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:30,950 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:31,539 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:31,950 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:32,070 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:32,950 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:33,950 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:34,950 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:35,951 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:36,951 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:37,225 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:37,542 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:37,951 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:38,951 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:39,951 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:40,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:41,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:42,922 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:42,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:43,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:44,560 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:06:44,918 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:06:44,919 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:06:44,961 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:45,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:46,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:47,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:48,006 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:48,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:49,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:50,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:51,956 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:52,430 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:06:52,430 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 with policy now +2025-01-18 21:06:52,433 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:06:52,433 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:06:52,433 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:06:52,434 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:06:52,557 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:06:52,558 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 +2025-01-18 21:06:52,558 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:06:52,694 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/6be1rgm0-media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 +2025-01-18 21:06:52,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:52,989 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/2gh50kbn-media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 +2025-01-18 21:06:53,454 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:53,558 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:53,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:54,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:55,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:56,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:57,559 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:57,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:58,871 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:58,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:59,925 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:06:59,925 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:06:59,965 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:00,968 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:01,560 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:01,966 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:02,975 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:03,972 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:04,065 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:04,974 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:05,973 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:06,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:07,207 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:07:07,207 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:07:07,207 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:07:07,562 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:07:07,563 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:07:07,968 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:08,978 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:09,117 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:09,563 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:09,982 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:10,974 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:11,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:12,972 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:13,565 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:13,973 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:14,284 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:14,560 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:07:14,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:07:14,920 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:07:14,977 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:15,565 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:15,973 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:16,973 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:17,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:18,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:19,932 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:19,975 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:20,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:21,975 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:22,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:23,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:24,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:25,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:25,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:26,978 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:27,977 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:28,977 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:29,923 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:07:29,923 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:07:29,977 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:30,977 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:31,006 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:31,977 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:32,978 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:33,978 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:34,978 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:35,978 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:36,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:36,981 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:37,979 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:38,979 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:39,980 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:40,992 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:41,097 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:07:41,097 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 with policy now +2025-01-18 21:07:41,098 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:07:41,098 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:07:41,099 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:07:41,100 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:07:41,367 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/29jfdqu6-media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 +2025-01-18 21:07:41,574 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:41,575 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:07:41,575 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 +2025-01-18 21:07:41,575 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:07:41,873 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/yjzneliy-media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 +2025-01-18 21:07:41,989 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:42,133 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:42,989 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:43,575 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:43,986 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:44,561 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:07:44,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:07:44,949 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:07:44,989 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:45,576 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:45,992 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:46,991 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:47,611 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:47,986 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:48,990 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:49,577 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:50,006 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:51,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:52,004 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:52,260 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:07:52,260 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:07:52,262 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:07:52,578 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:07:52,579 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:07:52,843 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/r19j16bi-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:07:52,876 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:53,009 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:53,579 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:54,004 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:55,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:55,579 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:56,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:56,999 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:57,580 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:57,999 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:58,247 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:58,999 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:59,923 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:07:59,923 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:07:59,999 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:00,999 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:01,999 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:03,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:04,000 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:04,077 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:05,000 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:06,000 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:07,000 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:08,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:09,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:09,932 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:10,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:11,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:12,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:13,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:14,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:14,561 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:08:14,923 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:08:14,923 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:08:15,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:15,012 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:16,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:17,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:18,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:19,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:20,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:20,933 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:21,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:22,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:23,005 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:24,009 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:24,117 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:08:24,117 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 with policy now +2025-01-18 21:08:24,123 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:08:24,123 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:08:24,123 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:08:24,124 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:08:24,382 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/4bra0h1v-media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 +2025-01-18 21:08:24,591 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:08:24,591 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 +2025-01-18 21:08:24,591 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:08:24,884 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/uxndxhhz-media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 +2025-01-18 21:08:25,021 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:25,591 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:08:25,941 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:26,022 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:27,023 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:28,024 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:29,020 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:29,592 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:08:29,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:08:29,928 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:08:30,028 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:31,013 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:31,593 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:08:31,709 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:32,026 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:33,028 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:33,594 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:08:34,026 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:34,312 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:08:34,313 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:08:34,314 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:08:34,594 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:08:34,594 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:08:35,029 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:35,595 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:08:36,030 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:36,941 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:37,041 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:38,049 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:39,042 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:39,596 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:08:40,040 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:41,041 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:42,029 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:42,513 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:43,030 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:43,597 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:08:44,030 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:44,561 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:08:44,927 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:08:44,927 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:08:45,030 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:46,030 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:47,030 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:48,031 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:48,036 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:49,031 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:50,031 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:51,031 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:52,031 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:53,031 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:53,940 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:54,032 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:55,032 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:56,032 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:57,032 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:58,032 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:59,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:59,927 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:08:59,927 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:08:59,994 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:00,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:01,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:02,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:03,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:04,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:05,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:05,940 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:06,034 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:07,034 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:08,034 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:09,034 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:10,042 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:10,477 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:09:10,478 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 with policy now +2025-01-18 21:09:10,486 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:09:10,486 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:09:10,486 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:09:10,487 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:09:10,607 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:09:10,608 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 +2025-01-18 21:09:10,608 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:09:10,751 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/2bl04fqg-media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 +2025-01-18 21:09:11,050 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:11,078 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/mxlj1ylo-media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 +2025-01-18 21:09:11,534 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:11,608 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:12,047 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:13,053 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:14,061 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:14,562 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:09:14,955 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:09:14,955 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:09:15,056 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:15,609 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:16,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:17,026 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:17,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:18,092 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:19,082 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:19,611 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:20,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:21,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:22,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:22,378 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:09:22,378 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:09:22,378 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:22,378 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:09:22,612 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:09:22,612 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:09:22,873 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/c7lpznkb-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:09:23,095 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:23,612 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:24,092 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:25,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:25,613 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:26,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:27,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:27,614 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:28,086 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:28,321 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:29,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:29,615 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:29,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:09:29,930 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:09:30,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:31,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:32,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:33,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:34,015 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:34,086 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:35,086 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:36,086 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:37,086 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:38,086 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:39,086 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:39,948 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:40,088 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:41,089 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:42,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:43,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:44,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:44,562 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:09:44,570 WARNING FileStreamThread:2577265 [file_stream.py:request_with_retry():674] requests_with_retry encountered retryable exception: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')). func: >, args: ('https://api.wandb.ai/files/serialexperimentsleon/extrinsic_contact_downstream/2t6elatx/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"eval/0_eval": {"_type": "video-file", "sha256": "373935c9a53d50a21516ae9864a2b682a53224d448bd3a5444b7c73d070560df", "size": 1151995, "path": "media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4"}, "global_step": 796, "_timestamp": 1737252513.5501094, "_runtime": 289.51256942749023, "_step": 12, "eval/1_eval": {"_type": "video-file", "sha256": "f136ddfa6a75f8c38319eb69e397cb0cf05992c189494f06c72a123d44e57cad", "size": 1169127, "path": "media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4"}, "eval/num_success": 6.0, "episode": 6.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.8098405599594116, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.780625581741333, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6978645324707031, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.7496918439865112, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "9019146f9223df893e271a59c4c8538311ba44dc2fd24a72c9dd4b2c64812680", "size": 1170383, "path": "media/videos/eval/2_eval_2_9019146f9223df893e27.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "a9352300f4a1f2d4fc59db4120a6eeb7a174fd0b71713a6348112847f63b66a3", "size": 1198800, "path": "media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "ff051e1686bf761b709a2c0ca3d274928506972c8da93aa7332f58ee411cb53a", "size": 1236656, "path": "media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "1a0fa497056b618ea0c00d6fc66b9c6ec244aca7406d79762d07999bb1f21f05", "size": 1158233, "path": "media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "a90b503bf7614a8b51c57e646cac1e6c87e69c6f37fa340d991743bbf1419c98", "size": 1334222, "path": "media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4"}}']}, 'wandb-history.jsonl': {'offset': 12, 'content': ['{"global_step": 796, "_timestamp": 1737252513.5501094, "eval/num_success": 6.0, "episode": 6.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.8098405599594116, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.780625581741333, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6978645324707031, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.7496918439865112, "eval/success_rate_ub_0.1": 1.0, "_runtime": 289.51256942749023, "_step": 12}']}, 'wandb-events.jsonl': {'offset': 9, 'content': ['{"system.proc.memory.availableMB": 54499.66, "system.memory": 15.13, "system.proc.memory.rssMB": 940.93, "system.proc.memory.percent": 1.47, "system.cpu": 13.47, "system.cpu.0.cpu_percent": 23.32, "system.cpu.1.cpu_percent": 23.09, "system.cpu.2.cpu_percent": 49.47, "system.cpu.3.cpu_percent": 26.25, "system.cpu.4.cpu_percent": 27.35, "system.cpu.5.cpu_percent": 24.99, "system.cpu.6.cpu_percent": 24.06, "system.cpu.7.cpu_percent": 24.03, "system.proc.cpu.threads": 59, "system.network.sent": 41607873249.27, "system.network.recv": 28071465410.33, "system.disk.\\\\.usagePercent": 76.67, "system.disk.\\\\.usageGB": 665.91, "system.disk.in": 0.27, "system.disk.out": 7344.52, "system.gpu.0.memoryAllocated": 28.37, "system.gpu.0.memoryAllocatedBytes": 2437205196.8, "system.gpu.0.memory": 4.0, "system.gpu.0.gpu": 13.13, "system.gpu.0.temp": 47.73, "system.gpu.0.powerWatts": 49.35, "system.gpu.0.powerPercent": 22.63, "_wandb": true, "_timestamp": 1737252554.561908, "_runtime": 330.524368}', '{"system.proc.memory.availableMB": 54431.22, "system.memory": 15.25, "system.proc.memory.rssMB": 933.58, "system.proc.memory.percent": 1.45, "system.cpu": 7.43, "system.cpu.0.cpu_percent": 31.45, "system.cpu.1.cpu_percent": 29.58, "system.cpu.2.cpu_percent": 47.41, "system.cpu.3.cpu_percent": 30.83, "system.cpu.4.cpu_percent": 27.8, "system.cpu.5.cpu_percent": 29.28, "system.cpu.6.cpu_percent": 26.15, "system.cpu.7.cpu_percent": 23.95, "system.proc.cpu.threads": 56, "system.network.sent": 46013138381.0, "system.network.recv": 31047672563.0, "system.disk.\\\\.usagePercent": 76.75, "system.disk.\\\\.usageGB": 666.75, "system.disk.in": 0.29, "system.disk.out": 8135.24, "system.gpu.0.memoryAllocated": 28.37, "system.gpu.0.memoryAllocatedBytes": 2436934314.67, "system.gpu.0.memory": 8.73, "system.gpu.0.gpu": 31.4, "system.gpu.0.temp": 49.8, "system.gpu.0.powerWatts": 57.08, "system.gpu.0.powerPercent": 26.55, "_wandb": true, "_timestamp": 1737252584.56238, "_runtime": 360.52484}']}, 'output.log': {'offset': 327, 'content': ['2025-01-19T02:09:11.352145 [INFO] [1737252549.414495]: Storing episode...\n', '2025-01-19T02:09:11.352145 [rosbag_record-21] killing on exit\n', '2025-01-19T02:09:11.352145 [print_text-20] killing on exit\n', '2025-01-19T02:09:11.352145 [pub_text-19] killing on exit\n', '2025-01-19T02:09:11.352145 [INFO] [1737252550.184000]: Stored episode 7.\n', '2025-01-19T02:09:11.352145 [INFO] [1737252550.186327]: Saving video...\n', '2025-01-19T02:09:11.352145 [INFO] [1737252550.399846]: Video saved!\n', '2025-01-19T02:09:11.352145 Starting episode 7\n', '2025-01-19T02:09:11.352145 [INFO] [1737252550.493449]: resetting environment\n', '2025-01-19T02:09:11.352145 [INFO] [1737252550.525712]: cleared current plan\n', '2025-01-19T02:09:11.352145 [INFO] [1737252550.533201]: moving to home\n', '2025-01-19T02:09:15.355592 [INFO] [1737252554.146813]: reached home\n', '2025-01-19T02:09:15.355592 [INFO] [1737252554.166841]: reset action history\n', '2025-01-19T02:09:19.358922 [INFO] [1737252557.025628]: environment reset\n', '2025-01-19T02:09:19.358922 Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success\n', '2025-01-19T02:09:23.364233 proceeding to start episode!\n', '2025-01-19T02:09:23.364233 Comuting lower confidence bound\n', '2025-01-19T02:09:23.364233 Comuting upper confidence bound\n', '2025-01-19T02:09:23.364233 Comuting lower confidence bound\n', '2025-01-19T02:09:23.364233 Comuting upper confidence bound\n', '2025-01-19T02:09:23.364233 Comuting lower confidence bound\n', '2025-01-19T02:09:23.364233 Comuting upper confidence bound\n', '2025-01-19T02:09:23.364233 Comuting lower confidence bound\n', '2025-01-19T02:09:23.364233 Comuting upper confidence bound\n', '2025-01-19T02:09:23.364233 [INFO] [1737252561.894930]: resetting environment\n', '2025-01-19T02:09:23.364233 [INFO] [1737252561.926678]: cleared current plan\n', '2025-01-19T02:09:23.364233 [INFO] [1737252561.928515]: moving to home\n', '2025-01-19T02:09:25.365484 [INFO] [1737252562.937209]: reached home\n', '2025-01-19T02:09:25.365484 [INFO] [1737252562.946625]: reset action history\n', '2025-01-19T02:09:27.367948 [INFO] [1737252565.826750]: environment reset\n', '2025-01-19T02:09:27.367948 bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_7_2025-01-18-21-09-25.bag\n', '2025-01-19T02:09:27.367948 ... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log\n', '2025-01-19T02:09:29.370300 \x1b[1mstarted roslaunch server http://158.130.50.37:38859/\n', '2025-01-19T02:09:29.370300 SUMMARY\n', '2025-01-19T02:09:29.370300 ========\n', '2025-01-19T02:09:29.370300 PARAMETERS\n', '2025-01-19T02:09:29.370300 * /rosdistro: noetic\n', '2025-01-19T02:09:29.370300 * /rosversion: 1.16.0\n', '2025-01-19T02:09:29.370300 NODES\n', '2025-01-19T02:09:29.370300 /\n', '2025-01-19T02:09:29.370300 print_text (rostopic/rostopic)\n', '2025-01-19T02:09:29.370300 pub_text (rostopic/rostopic)\n', '2025-01-19T02:09:29.370300 rosbag_record (rosbag/record)\n', '2025-01-19T02:09:29.370300 \x1b[1mROS_MASTER_URI=http://localhost:11311\n', '2025-01-19T02:09:29.370300 \x1b[1mprocess[pub_text-22]: started with pid [2578397]\n', '2025-01-19T02:09:29.370300 \x1b[1mprocess[print_text-23]: started with pid [2578398]\n', '2025-01-19T02:09:29.370300 \x1b[1mprocess[rosbag_record-24]: started with pid [2578399]\n', '2025-01-19T02:09:29.370300 started bagging!\n']}}, 'dropped': 0}} +2025-01-18 21:09:44,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:09:44,930 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:09:45,077 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:45,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:46,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:47,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:48,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:49,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:50,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:50,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:51,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:52,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:53,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:54,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:55,117 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:55,175 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:09:55,176 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 with policy now +2025-01-18 21:09:55,217 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:09:55,217 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:09:55,218 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:09:55,218 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:09:55,471 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/ob9870et-media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 +2025-01-18 21:09:55,624 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:55,624 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:09:55,624 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 +2025-01-18 21:09:55,625 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:09:55,919 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/kqy5bfoe-media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 +2025-01-18 21:09:56,116 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:56,258 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:57,111 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:57,625 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:58,112 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:59,116 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:59,626 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:59,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:09:59,949 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:10:00,117 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:01,117 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:01,794 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:02,109 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:03,114 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:03,627 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:04,153 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:04,429 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:10:04,430 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:10:04,430 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:10:04,627 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:10:04,628 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:10:05,115 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:05,628 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:06,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:07,138 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:07,317 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:07,628 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:08,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:09,143 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:09,629 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:10,125 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:11,132 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:12,133 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:12,970 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:13,144 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:14,127 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:14,563 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:10:14,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:10:14,929 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:10:15,127 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:15,632 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:16,128 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:17,128 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:17,633 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:18,105 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:18,128 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:19,128 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:19,633 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:20,128 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:21,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:22,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:23,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:23,956 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:24,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:25,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:26,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:27,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:27,636 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:28,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:29,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:29,637 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:29,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:10:29,929 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:10:30,045 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:30,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:31,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:32,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:33,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:33,639 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:34,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:35,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:35,639 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:35,956 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:36,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:37,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:38,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:39,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:40,131 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:41,140 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:41,497 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:41,642 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:41,827 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:10:41,827 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 with policy now +2025-01-18 21:10:41,837 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:10:41,838 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:10:41,838 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:10:41,838 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:10:42,116 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/cwk3e2tk-media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 +2025-01-18 21:10:42,144 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:42,642 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:10:42,642 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 +2025-01-18 21:10:42,642 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:10:42,958 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/2hglpkjk-media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 +2025-01-18 21:10:43,150 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:43,642 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:44,167 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:44,563 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:10:44,982 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:10:44,982 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:10:45,154 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:46,165 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:46,553 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:47,211 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:47,644 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:48,173 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:49,178 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:49,487 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:10:49,487 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:10:49,492 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:10:49,645 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:49,645 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:10:49,645 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:10:49,903 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/z2qrvsct-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:10:50,167 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:51,171 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:51,646 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:51,982 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:52,189 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:53,173 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:53,646 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:54,195 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:55,180 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:56,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:57,196 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:57,974 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:58,199 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:59,192 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:59,649 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:59,935 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:10:59,935 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:11:00,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:01,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:01,650 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:02,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:03,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:03,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:04,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:05,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:05,652 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:06,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:07,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:07,652 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:08,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:08,245 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:09,195 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:10,195 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:11,195 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:12,195 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:13,195 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:13,961 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:14,196 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:14,564 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:11:14,935 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:11:14,936 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:11:15,196 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:16,196 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:17,196 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:18,196 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:19,095 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:19,196 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:20,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:21,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:21,657 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:22,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:23,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:24,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:24,458 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:25,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:26,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:27,221 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:27,660 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:27,811 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:11:27,811 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 with policy now +2025-01-18 21:11:27,814 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:11:27,814 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:11:27,814 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:11:27,814 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:11:28,082 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/0391rdg1-media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 +2025-01-18 21:11:28,209 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:28,660 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:11:28,661 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 +2025-01-18 21:11:28,661 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:11:28,977 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/qrkurpbu-media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 +2025-01-18 21:11:29,214 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:29,661 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:29,846 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:29,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:11:29,959 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:11:30,217 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:31,214 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:32,224 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:33,230 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:33,662 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:34,260 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:34,962 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:35,225 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:35,663 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:36,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:37,221 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:37,544 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:11:37,544 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:11:37,545 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:11:37,664 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:37,664 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:11:37,664 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:11:38,261 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:39,242 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:39,665 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:39,967 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:40,244 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:41,233 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:41,665 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:42,239 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:43,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:43,666 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:44,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:44,564 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:11:44,940 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:11:44,940 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:11:45,063 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:45,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:46,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:47,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:48,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:49,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:50,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:50,964 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:51,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:52,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:53,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:54,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:55,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:55,965 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:56,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:57,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:58,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:59,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:59,940 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:11:59,941 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:12:00,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:01,084 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:01,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:02,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:03,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:04,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:05,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:06,238 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:06,965 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:07,238 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:08,238 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:09,238 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:10,246 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:10,269 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:12:10,269 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 with policy now +2025-01-18 21:12:10,313 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:12:10,313 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:12:10,314 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:12:10,314 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:12:10,535 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/2n9jc6ae-media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 +2025-01-18 21:12:10,677 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:12:10,677 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 +2025-01-18 21:12:10,678 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:12:10,979 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/i301d7bv-media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 +2025-01-18 21:12:11,257 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:11,678 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:12,277 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:12,339 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:13,291 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:14,264 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:14,565 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:12:14,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:12:14,954 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:12:15,267 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:15,679 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:16,271 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:17,265 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:17,680 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:18,097 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:18,270 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:19,274 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:20,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:21,302 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:22,305 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:22,600 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:12:22,600 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:12:22,601 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:12:22,682 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:12:22,682 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:12:22,953 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/szyye1o9-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:12:23,284 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:23,566 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:23,682 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:24,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:25,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:25,683 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:26,299 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:27,299 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:27,684 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:28,314 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:28,705 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:29,296 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:29,971 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:12:29,972 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:12:30,308 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:31,309 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:32,299 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:33,297 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:34,066 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:34,300 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:35,293 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:35,687 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:36,294 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:37,294 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:37,688 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:38,294 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:39,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:39,294 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:40,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:41,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:42,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:43,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:44,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:44,517 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:44,565 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:12:44,945 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:12:44,945 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:12:45,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:45,691 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:46,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:47,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:48,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:49,296 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:49,988 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:50,296 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:51,296 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:52,296 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:53,296 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:54,296 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:54,988 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:55,298 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:56,307 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:56,448 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:12:56,448 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 with policy now +2025-01-18 21:12:56,451 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:12:56,451 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:12:56,451 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:12:56,452 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:12:56,696 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:12:56,696 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 +2025-01-18 21:12:56,696 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:12:56,718 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/nyybbuco-media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 +2025-01-18 21:12:57,015 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/ykakra7a-media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 +2025-01-18 21:12:57,305 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:57,696 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:58,346 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:59,359 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:59,966 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:12:59,967 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:13:00,083 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:00,335 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:01,356 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:01,698 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:02,361 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:03,381 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:04,368 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:04,652 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:13:04,652 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:13:04,653 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:13:04,699 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:13:04,699 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:13:05,395 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:05,442 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:05,699 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:06,382 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:07,390 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:07,700 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:08,400 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:09,407 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:09,701 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:10,425 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:11,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:11,406 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:12,428 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:13,433 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:14,429 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:14,566 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:13:14,988 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:13:14,988 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:13:15,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:16,105 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:16,420 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:17,461 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:17,704 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:18,420 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:19,420 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:19,705 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:20,420 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:21,420 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:21,995 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:22,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:23,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:24,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:25,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:26,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:26,995 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:27,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:28,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:29,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:29,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:13:29,960 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:13:30,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:31,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:32,094 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:32,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:33,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:34,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:35,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:36,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:37,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:37,995 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:38,425 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:39,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:40,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:41,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:42,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:42,995 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:43,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:44,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:44,566 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:13:44,974 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:13:44,975 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:13:45,419 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:13:45,419 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 with policy now +2025-01-18 21:13:45,439 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:45,444 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:13:45,444 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:13:45,444 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:13:45,445 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:13:45,702 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/qc4ilx14-media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 +2025-01-18 21:13:45,715 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:45,715 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:13:45,715 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 +2025-01-18 21:13:45,716 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:13:46,020 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/0ea2c26n-media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 +2025-01-18 21:13:46,445 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:47,438 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:47,716 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:48,467 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:48,507 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:49,499 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:49,717 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:50,457 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:51,455 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:52,439 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:53,472 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:53,718 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:53,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:54,458 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:55,468 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:56,457 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:57,470 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:58,475 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:59,017 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:59,477 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:59,998 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:13:59,998 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:14:00,477 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:01,482 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:02,515 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:03,505 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:03,722 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:14:03,722 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:14:03,723 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:14:03,723 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:14:03,724 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:14:04,524 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:04,724 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:14:04,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:05,492 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:05,724 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:06,515 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:07,497 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:08,523 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:09,548 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:09,726 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:09,837 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:10,525 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:11,551 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:12,522 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:13,540 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:14,545 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:14,567 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:14:14,971 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:14:14,972 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:14:15,126 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:15,561 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:16,571 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:17,538 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:17,729 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:18,538 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:19,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:19,730 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:20,538 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:21,060 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:21,538 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:21,731 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:22,538 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:23,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:24,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:25,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:26,061 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:26,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:27,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:28,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:29,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:29,963 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:14:29,963 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:14:30,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:31,098 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:31,540 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:32,540 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:33,540 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:34,540 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:35,540 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:36,540 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:37,061 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:37,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:38,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:39,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:40,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:41,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:42,061 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:42,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:43,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:44,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:44,567 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:14:44,963 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:14:44,963 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:14:45,577 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:45,741 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:46,154 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:14:46,154 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 with policy now +2025-01-18 21:14:46,157 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:14:46,157 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:14:46,157 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:14:46,158 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:14:46,444 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/w5wwaed3-media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 +2025-01-18 21:14:46,555 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:46,741 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:14:46,741 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 +2025-01-18 21:14:46,742 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:14:47,056 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/wnenej0g-media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 +2025-01-18 21:14:47,206 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:47,588 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:47,742 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:48,615 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:49,587 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:50,593 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:51,609 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:51,743 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:52,600 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:52,965 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:53,602 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:53,744 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:54,608 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:55,630 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:56,607 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:57,624 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:58,081 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:58,609 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:59,643 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:59,969 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:14:59,969 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:15:00,621 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:01,633 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:02,636 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:03,096 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:03,652 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:04,670 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:05,672 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:06,649 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:07,657 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:08,658 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:09,073 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:09,672 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:10,665 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:11,675 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:12,666 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:13,665 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:13,752 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:15:13,752 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:15:13,808 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:15:13,809 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:15:13,809 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:15:14,388 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:14,436 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/yyx1h8uq-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:15:14,567 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:15:14,692 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:14,753 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:15:14,978 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:15:14,978 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:15:15,723 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:15,753 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:15:16,690 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:17,688 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:18,720 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:19,569 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:19,713 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:19,755 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:15:20,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:21,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:22,715 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:23,727 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:24,735 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:25,116 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:25,761 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:26,801 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:27,777 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:28,728 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:29,728 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:29,759 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:15:29,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:15:29,976 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:15:30,728 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:31,102 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:31,728 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:31,760 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:15:32,728 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:33,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:34,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:35,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:36,102 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:36,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:37,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:38,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:39,732 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:40,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:41,103 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:41,730 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:42,730 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:43,730 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:44,568 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:15:44,730 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:44,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:15:44,976 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:15:45,730 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:46,118 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:46,730 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:47,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:48,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:49,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:50,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:51,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:52,103 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:52,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:53,732 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:54,733 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:55,733 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:56,752 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:57,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:57,363 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:15:57,363 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 with policy now +2025-01-18 21:15:57,377 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:15:57,378 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:15:57,378 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:15:57,378 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:15:57,643 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/p9egipbp-media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 +2025-01-18 21:15:57,739 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:57,770 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:15:57,770 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:15:57,771 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 +2025-01-18 21:15:57,771 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:15:58,066 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/pnz7oe18-media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 +2025-01-18 21:15:58,762 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:59,752 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:59,771 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:15:59,991 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:15:59,992 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:16:00,755 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:01,772 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:01,778 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:02,757 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:03,009 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:03,769 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:04,779 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:05,774 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:05,819 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:06,808 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:07,774 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:07,775 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:16:07,785 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:07,880 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:16:07,880 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:16:07,881 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:16:08,357 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:08,775 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:16:08,835 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:09,775 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:09,783 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:10,820 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:11,783 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:12,790 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:13,737 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:13,779 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:13,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:14,568 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:16:14,822 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:14,981 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:16:14,982 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:16:15,810 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:16,841 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:17,837 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:18,824 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:18,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:19,781 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:19,810 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:20,810 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:21,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:22,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:23,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:24,122 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:24,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:25,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:26,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:27,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:28,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:29,122 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:29,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:29,979 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:16:29,979 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:16:30,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:31,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:32,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:33,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:34,123 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:34,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:35,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:36,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:37,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:38,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:39,123 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:39,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:40,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:41,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:42,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:43,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:44,569 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:16:44,570 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:44,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:44,979 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:16:44,979 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:16:45,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:46,831 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:47,097 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:16:47,097 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 with policy now +2025-01-18 21:16:47,117 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:16:47,117 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:16:47,117 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:16:47,118 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:16:47,368 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/psj3ohm0-media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 +2025-01-18 21:16:47,797 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:47,797 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:16:47,797 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 +2025-01-18 21:16:47,797 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:16:47,864 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:48,117 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/y5yntkla-media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 +2025-01-18 21:16:48,854 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:49,849 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:50,127 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:50,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:51,798 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:51,851 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:52,854 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:53,848 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:54,847 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:55,139 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:55,860 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:56,800 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:56,884 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:57,877 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:58,903 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:59,903 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:00,029 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:17:00,029 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:17:00,902 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:01,138 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:01,941 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:02,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:03,803 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:17:03,948 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:03,948 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:17:03,948 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:17:03,948 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:17:04,803 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:04,804 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:17:04,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:05,935 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:06,804 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:06,936 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:07,138 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:07,923 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:08,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:09,935 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:10,806 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:10,948 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:11,933 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:12,167 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:12,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:13,969 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:14,569 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:17:14,998 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:17:14,998 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:17:15,045 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:16,010 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:16,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:17,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:17,963 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:18,809 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:18,956 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:19,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:20,810 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:20,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:21,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:22,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:23,103 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:23,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:24,812 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:24,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:25,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:26,813 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:26,958 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:27,958 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:28,545 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:28,814 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:28,958 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:29,958 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:29,985 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:17:29,985 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:17:30,958 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:31,958 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:32,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:33,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:34,131 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:34,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:35,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:36,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:37,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:38,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:39,131 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:39,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:40,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:41,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:42,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:43,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:44,131 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:44,570 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:17:44,993 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:45,004 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:17:45,004 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:17:45,975 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:45,998 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:17:45,998 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 with policy now +2025-01-18 21:17:46,041 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:17:46,042 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:17:46,042 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:17:46,046 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:17:46,296 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/05pnzh1z-media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 +2025-01-18 21:17:46,822 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:46,822 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:17:46,822 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 +2025-01-18 21:17:46,822 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:17:46,973 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:47,145 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/xnrnd6np-media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 +2025-01-18 21:17:47,822 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:47,997 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:48,823 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:49,004 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:49,734 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:49,995 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:50,984 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:52,026 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:52,824 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:53,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:54,027 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:54,825 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:55,043 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:55,172 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:56,071 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:57,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:58,046 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:59,046 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:59,994 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:17:59,994 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:18:00,055 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:00,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:00,828 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:18:01,021 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:18:01,021 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:18:01,021 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:18:01,022 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:01,089 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/tncyld2n-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:18:01,828 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:18:02,024 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:02,829 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:03,067 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:04,030 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:04,829 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:05,044 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:06,078 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:06,184 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:06,830 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:07,054 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:08,059 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:09,077 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:10,054 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:11,081 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:12,076 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:12,182 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:13,084 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:14,073 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:14,570 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:18:15,067 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:18:15,067 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:18:15,109 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:16,147 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:17,113 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:18,107 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:18,179 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:19,089 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:20,089 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:20,836 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:21,089 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:22,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:22,838 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:23,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:24,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:24,152 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:25,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:26,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:27,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:28,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:29,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:29,152 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:29,988 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:18:29,988 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:18:30,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:31,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:32,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:33,094 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:34,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:34,181 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:35,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:36,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:37,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:38,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:39,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:40,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:40,151 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:41,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:42,094 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:43,094 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:44,094 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:44,571 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:18:44,989 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:18:44,989 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:18:45,094 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:45,152 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:46,094 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:47,114 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:47,603 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:18:47,603 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 with policy now +2025-01-18 21:18:47,609 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:18:47,610 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:18:47,610 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:18:47,610 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:18:47,850 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:18:47,850 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 +2025-01-18 21:18:47,850 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:18:47,933 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/p7cj1ipe-media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 +2025-01-18 21:18:48,120 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:48,231 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/yi5afswe-media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 +2025-01-18 21:18:48,850 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:49,127 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:50,152 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:50,155 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:50,851 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:51,177 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:52,149 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:52,852 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:53,184 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:54,171 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:55,157 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:55,158 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:56,188 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:56,854 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:57,186 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:58,199 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:59,226 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:00,012 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:19:00,012 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:19:00,181 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:00,216 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:01,184 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:02,186 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:03,235 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:04,198 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:05,208 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:06,180 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:06,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:06,858 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:19:07,099 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:19:07,099 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:19:07,100 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:19:07,208 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:07,858 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:19:07,859 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:19:08,215 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:08,859 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:09,243 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:10,229 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:10,860 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:11,223 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:11,428 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:12,232 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:12,861 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:13,235 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:14,267 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:14,577 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:19:15,059 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:19:15,059 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:19:15,283 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:16,268 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:17,172 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:17,259 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:18,269 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:19,260 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:20,287 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:21,262 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:22,191 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:22,330 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:23,308 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:24,274 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:24,866 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:25,274 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:26,274 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:26,867 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:27,277 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:28,180 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:28,275 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:29,275 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:29,995 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:19:29,995 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:19:30,275 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:31,276 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:32,276 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:33,181 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:33,276 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:34,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:35,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:36,279 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:37,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:38,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:39,181 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:39,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:40,281 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:41,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:42,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:43,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:44,181 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:44,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:44,571 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:19:44,999 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:19:44,999 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:19:45,281 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:46,282 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:47,285 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:48,281 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:49,181 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:49,289 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:50,285 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:51,286 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:52,326 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:53,090 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:19:53,090 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 with policy now +2025-01-18 21:19:53,101 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:19:53,101 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:19:53,101 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:19:53,103 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:19:53,328 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:53,371 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/cy0ojnxt-media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 +2025-01-18 21:19:53,879 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:53,879 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:19:53,879 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 +2025-01-18 21:19:53,879 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:19:54,239 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:54,245 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/s7eojymz-media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 +2025-01-18 21:19:54,343 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:54,879 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:55,331 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:56,344 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:57,326 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:58,343 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:58,881 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:59,307 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:59,913 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:00,036 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:20:00,036 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:20:00,306 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:01,330 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:02,342 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:02,883 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:03,381 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:04,369 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:05,198 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:05,356 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:06,396 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:07,436 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:07,885 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:20:08,171 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:20:08,171 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:20:08,172 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:20:08,381 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:08,886 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:08,886 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:20:08,886 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:20:09,382 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:10,247 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:10,395 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:10,886 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:11,386 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:12,389 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:13,415 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:14,410 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:14,572 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:20:14,888 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:15,020 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:20:15,020 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:20:15,251 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:15,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:16,404 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:17,443 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:18,416 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:19,416 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:20,449 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:21,215 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:21,448 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:22,442 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:23,447 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:24,445 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:25,434 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:26,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:26,893 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:26,915 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:27,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:28,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:28,894 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:29,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:30,011 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:20:30,012 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:20:30,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:31,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:32,212 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:32,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:33,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:34,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:35,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:36,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:37,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:38,213 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:38,425 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:39,425 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:40,425 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:41,425 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:42,426 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:42,901 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:43,293 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:43,426 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:44,426 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:44,572 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:20:45,011 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:20:45,012 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:20:45,426 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:46,426 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:47,427 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:48,427 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:49,213 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:49,426 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:50,427 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:51,427 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:52,427 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:53,486 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:53,965 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:20:53,966 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 with policy now +2025-01-18 21:20:53,987 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:20:53,988 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:20:53,988 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:20:53,988 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:20:54,277 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/g1t2eo87-media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 +2025-01-18 21:20:54,438 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:54,906 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:54,907 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:20:54,907 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 +2025-01-18 21:20:54,907 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:20:55,026 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:55,220 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/pvmpldhg-media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 +2025-01-18 21:20:55,478 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:55,907 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:56,467 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:56,907 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:57,496 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:58,489 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:58,908 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:59,493 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:00,020 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:21:00,020 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:21:00,115 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:00,517 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:00,909 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:21:01,518 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:02,512 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:02,910 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:21:03,531 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:04,548 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:05,285 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:05,544 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:06,554 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:07,553 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:08,585 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:09,243 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:21:09,244 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:09,244 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:09,613 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:09,913 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:21:09,913 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:21:10,609 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:10,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 21:21:10,908 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 21:21:10,914 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:21:11,137 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:21:11,342 INFO wandb-upload_2:2577265 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp2x7lu8zi +2025-01-18 21:21:11,593 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:11,744 INFO SenderThread:2577265 [sender.py:send_request_log_artifact():1525] logged artifact run-2t6elatx-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MjA0NTYxNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzExNTAzNA==', 'latestArtifact': None}} +2025-01-18 21:21:11,744 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:21:11,744 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json with policy now +2025-01-18 21:21:11,745 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:11,745 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:21:11,745 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:11,746 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:11,914 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:21:11,914 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json +2025-01-18 21:21:11,914 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media +2025-01-18 21:21:11,915 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table +2025-01-18 21:21:11,915 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval +2025-01-18 21:21:11,940 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/hzqzkb5b-media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json +2025-01-18 21:21:12,206 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/e7jjem0a-media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json +2025-01-18 21:21:12,593 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:13,594 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:14,573 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:21:14,594 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:15,016 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:21:15,016 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:21:15,595 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:16,596 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:17,233 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:17,595 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:18,596 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:19,596 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:20,596 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:21,596 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:22,597 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:22,893 DEBUG SenderThread:2577265 [sender.py:send():379] send: telemetry +2025-01-18 21:21:22,894 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:22,919 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:21:22,941 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,941 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,941 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,942 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,942 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,942 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,943 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,943 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,943 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,944 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,944 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,944 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,945 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,945 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,945 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,946 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,946 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,946 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,946 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,947 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,947 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,947 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:23,034 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 21:21:23,034 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 21:21:23,044 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:21:23,188 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/ahcq43fd-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:21:23,382 INFO wandb-upload_2:2577265 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpcblxnubg +2025-01-18 21:21:23,597 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:23,800 INFO SenderThread:2577265 [sender.py:send_request_log_artifact():1525] logged artifact run-2t6elatx-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MjA0NTczOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzExNTA1Nw==', 'latestArtifact': None}} +2025-01-18 21:21:23,800 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:21:23,800 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json with policy now +2025-01-18 21:21:23,800 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:21:23,800 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:23,801 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:23,920 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:21:23,921 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:21:23,921 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json +2025-01-18 21:21:23,921 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval +2025-01-18 21:21:23,999 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/mr0gm6ct-media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json +2025-01-18 21:21:24,280 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/d6rv1i6i-media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json +2025-01-18 21:21:24,598 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:24,921 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:21:25,597 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:26,597 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:27,598 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:28,233 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:28,599 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:29,597 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:29,689 DEBUG SenderThread:2577265 [sender.py:send():379] send: exit +2025-01-18 21:21:29,689 INFO SenderThread:2577265 [sender.py:send_exit():586] handling exit code: 0 +2025-01-18 21:21:29,690 INFO SenderThread:2577265 [sender.py:send_exit():588] handling runtime: 1065 +2025-01-18 21:21:29,691 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:29,691 INFO SenderThread:2577265 [sender.py:send_exit():594] send defer +2025-01-18 21:21:29,691 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:29,691 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-18 21:21:29,691 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:29,691 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-18 21:21:29,691 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 1 +2025-01-18 21:21:29,691 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:29,691 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-18 21:21:29,692 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:29,692 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-18 21:21:29,692 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 2 +2025-01-18 21:21:29,692 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:29,692 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-18 21:21:29,692 INFO HandlerThread:2577265 [system_monitor.py:finish():203] Stopping system monitor +2025-01-18 21:21:29,692 DEBUG SystemMonitor:2577265 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-18 21:21:29,692 DEBUG SystemMonitor:2577265 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-18 21:21:29,693 INFO HandlerThread:2577265 [interfaces.py:finish():200] Joined cpu monitor +2025-01-18 21:21:29,693 INFO HandlerThread:2577265 [interfaces.py:finish():200] Joined disk monitor +2025-01-18 21:21:29,744 INFO HandlerThread:2577265 [interfaces.py:finish():200] Joined gpu monitor +2025-01-18 21:21:29,744 INFO HandlerThread:2577265 [interfaces.py:finish():200] Joined memory monitor +2025-01-18 21:21:29,744 INFO HandlerThread:2577265 [interfaces.py:finish():200] Joined network monitor +2025-01-18 21:21:29,744 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:29,744 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-18 21:21:29,744 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 3 +2025-01-18 21:21:29,745 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:21:29,745 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:29,745 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-18 21:21:29,745 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:29,745 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-18 21:21:29,745 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 4 +2025-01-18 21:21:29,745 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:29,745 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-18 21:21:29,923 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:21:30,924 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:21:36,270 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:21:36,270 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:36,270 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: keepalive +2025-01-18 21:21:36,271 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:36,271 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-18 21:21:36,271 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 5 +2025-01-18 21:21:36,271 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:21:36,271 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:21:36,271 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:36,272 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:36,272 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-18 21:21:36,272 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:36,272 DEBUG SenderThread:2577265 [sender.py:send():379] send: summary +2025-01-18 21:21:36,276 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:36,277 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:36,277 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-18 21:21:36,277 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 6 +2025-01-18 21:21:36,277 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:36,277 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-18 21:21:36,277 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:36,277 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-18 21:21:36,283 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:36,426 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 7 +2025-01-18 21:21:36,427 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:36,427 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-18 21:21:36,427 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:36,427 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-18 21:21:36,691 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:21:36,927 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/config.yaml +2025-01-18 21:21:36,927 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:21:39,909 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 8 +2025-01-18 21:21:39,909 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:21:39,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:39,910 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-18 21:21:39,910 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:39,910 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-18 21:21:39,910 INFO SenderThread:2577265 [job_builder.py:build():440] Attempting to build job artifact +2025-01-18 21:21:39,913 INFO SenderThread:2577265 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-18 21:21:39,915 INFO SenderThread:2577265 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-18 21:21:39,928 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:21:39,934 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 9 +2025-01-18 21:21:39,934 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:39,934 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-18 21:21:39,935 DEBUG SenderThread:2577265 [sender.py:send():379] send: artifact +2025-01-18 21:21:40,322 INFO wandb-upload_1:2577265 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpe1wv_oo5 +2025-01-18 21:21:40,322 INFO wandb-upload_2:2577265 [upload_job.py:push():85] Skipped uploading /tmp/tmpfj0s81cv/wandb-job.json +2025-01-18 21:21:40,506 INFO wandb-upload_3:2577265 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp8gzxv2_4 +2025-01-18 21:21:40,692 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:21:41,186 INFO SenderThread:2577265 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1MjA0NTk0MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1MjAyMTE2MQ==', 'versionIndex': 450}}} +2025-01-18 21:21:41,186 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:41,186 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-18 21:21:41,186 INFO SenderThread:2577265 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-18 21:21:41,929 INFO SenderThread:2577265 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files +2025-01-18 21:21:41,929 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/config.yaml config.yaml +2025-01-18 21:21:41,930 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json wandb-summary.json +2025-01-18 21:21:41,930 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:21:41,930 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/requirements.txt requirements.txt +2025-01-18 21:21:41,930 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-metadata.json wandb-metadata.json +2025-01-18 21:21:41,930 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 21:21:41,930 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log output.log +2025-01-18 21:21:41,932 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/diff.patch diff.patch +2025-01-18 21:21:41,933 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-18 21:21:41,936 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 +2025-01-18 21:21:41,936 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 +2025-01-18 21:21:41,936 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json +2025-01-18 21:21:41,940 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 10 +2025-01-18 21:21:41,940 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:21:41,940 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:41,940 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-18 21:21:41,940 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:41,940 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-18 21:21:41,940 INFO SenderThread:2577265 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 21:21:42,131 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/config.yaml +2025-01-18 21:21:42,147 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:21:42,163 INFO wandb-upload_4:2577265 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:21:42,171 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/requirements.txt +2025-01-18 21:21:42,223 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/b03309oi-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:21:42,423 INFO Thread-11 (_thread_body):2577265 [sender.py:transition_state():614] send defer: 11 +2025-01-18 21:21:42,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:42,423 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-18 21:21:42,424 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:42,424 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-18 21:21:42,424 INFO SenderThread:2577265 [file_pusher.py:join():175] waiting for file pusher +2025-01-18 21:21:42,427 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 12 +2025-01-18 21:21:42,427 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:42,427 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-18 21:21:42,427 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:42,427 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-18 21:21:42,427 INFO SenderThread:2577265 [file_stream.py:finish():601] file stream finish called +2025-01-18 21:21:42,661 INFO SenderThread:2577265 [file_stream.py:finish():605] file stream finish is done +2025-01-18 21:21:42,661 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 13 +2025-01-18 21:21:42,661 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:42,661 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-18 21:21:42,661 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:42,661 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-18 21:21:42,661 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 14 +2025-01-18 21:21:42,662 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:42,662 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-18 21:21:42,662 DEBUG SenderThread:2577265 [sender.py:send():379] send: final +2025-01-18 21:21:42,662 DEBUG SenderThread:2577265 [sender.py:send():379] send: footer +2025-01-18 21:21:42,662 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:42,662 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-18 21:21:42,662 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:21:42,663 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:21:42,663 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: server_info +2025-01-18 21:21:42,663 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: server_info +2025-01-18 21:21:42,664 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:21:42,665 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: get_summary +2025-01-18 21:21:42,665 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-18 21:21:42,666 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:42,719 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:21:42,719 INFO MainThread:2577265 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-18 21:21:42,720 INFO MainThread:2577265 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-18 21:21:42,720 INFO MainThread:2577265 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-18 21:21:42,721 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: shutdown +2025-01-18 21:21:42,721 INFO HandlerThread:2577265 [handler.py:finish():882] shutting down handler +2025-01-18 21:21:43,665 INFO WriterThread:2577265 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/run-2t6elatx.wandb +2025-01-18 21:21:43,719 INFO SenderThread:2577265 [sender.py:finish():1615] shutting down sender +2025-01-18 21:21:43,719 INFO SenderThread:2577265 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 21:21:43,720 INFO SenderThread:2577265 [file_pusher.py:join():175] waiting for file pusher diff --git a/2t6elatx/wandb/debug.log b/2t6elatx/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..34d3c18d2893ce810a4a9160a32d1df30b8ee614 --- /dev/null +++ b/2t6elatx/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-18 21:03:44,031 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-18 21:03:44,031 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Configure stats pid to 2577081 +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/settings +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/logs/debug.log +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/logs/debug-internal.log +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_init.py:init():569] calling init triggers +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 2, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_2', 'wandb_notes': '3465_2_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 2, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx'} +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_init.py:init():619] starting backend +2025-01-18 21:03:44,033 INFO MainThread:2577081 [wandb_init.py:init():623] setting up manager +2025-01-18 21:03:44,036 INFO MainThread:2577081 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-18 21:03:44,037 INFO MainThread:2577081 [wandb_init.py:init():631] backend started and connected +2025-01-18 21:03:44,051 INFO MainThread:2577081 [wandb_init.py:init():720] updated telemetry +2025-01-18 21:03:44,059 INFO MainThread:2577081 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-18 21:03:44,500 INFO MainThread:2577081 [wandb_run.py:_on_init():2435] communicating current version +2025-01-18 21:03:44,541 INFO MainThread:2577081 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-18 21:03:44,541 INFO MainThread:2577081 [wandb_init.py:init():804] starting run threads in backend +2025-01-18 21:03:44,904 INFO MainThread:2577081 [wandb_run.py:_console_start():2413] atexit reg +2025-01-18 21:03:44,904 INFO MainThread:2577081 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-18 21:03:44,904 INFO MainThread:2577081 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-18 21:03:44,904 INFO MainThread:2577081 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-18 21:03:44,905 INFO MainThread:2577081 [wandb_init.py:init():847] run started, returning control to user process +2025-01-18 21:03:44,906 INFO MainThread:2577081 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/tb, True +2025-01-18 21:03:48,880 INFO MainThread:2577081 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-18 21:21:43,722 WARNING MsgRouterThr:2577081 [router.py:message_loop():77] message_loop has been closed diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/code/FISH/eval_robot.py b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/config.yaml b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..523dba03d260685ed9cacdd501b8a244d65afac6 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/config.yaml @@ -0,0 +1,927 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 2 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '3465_2' +wandb_notes: + desc: null + value: 3465_2_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: null +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 2 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737252224 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/diff.patch b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..3c1e7850f0918bff4b78c50e50ded316e53e70d6 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/diff.patch @@ -0,0 +1,102 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0b377ae..c11eb32 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -132,8 +132,18 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++# wandb_run_id: '3456_2' # seed 2 ++ ++# all books, 06/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3465_0' ++# wandb_run_id: '3465_1' # seed 1 ++wandb_run_id: '3465_2' # seed 2 ++ + # RGBD+mask+act history +-wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + +@@ -191,7 +201,7 @@ wandb_run_id: '1002_0' + true_action_history: false + wandb_notes: null + +-checkpoint_epoch: 12000 # for depth + all contact ++checkpoint_epoch: 7500 # for depth + all contact + load_residual_weight: false + + checkpoint_root_dir: '/home/${oc.env:USER}/fish_leon/FISH' +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..9509617 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,15 +167,21 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ # '3456_1', ++ # '3456_2', ++ '3465_0', ++ '3465_1', ++ '3465_2', + ] + +-checkpoint_epoch = 12000 ++checkpoint_epoch = 7500 + + # use subprocess to download the model checkpoints in parallel + process_list = [] +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json new file mode 100644 index 0000000000000000000000000000000000000000..40a7200787b4b3e42b061b26186780647c7673a5 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.75, 0.48087930679703694, 0.919547632252707, 0.5107601731994087, 0.9122562705645977, 0.5460197776463978, 0.886025808674378, 0.6174886487189397, 0.8676386735598093]]} \ No newline at end of file diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json new file mode 100644 index 0000000000000000000000000000000000000000..39a08797f8897bc81acee099e56b9ce224e83aa6 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[15, 20, 0.75, 0.510935544965464, 0.9235437958585514, 0.5362904965804953, 0.9076186566729699, 0.5726168081019127, 0.8630436881650583, 0.6052489131478573, 0.8673701099316663]]} \ No newline at end of file diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3b2695fe0c174de459d9baef2ad0da8a7c4fdf32 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:373935c9a53d50a21516ae9864a2b682a53224d448bd3a5444b7c73d070560df +size 1151995 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0cdb714b9d214b4f677523a9ead2078bb742eb8e --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:194dcd6049c4233da56c1320d7fff1f54a4b9ebb30a13b10b4abae786a908461 +size 1191627 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9a195cbb4d8ef55cb90fd9c89d86c2794d7a82a5 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:136a95e750557f39dd091db067809586d46c1938c60d2ff38e1ecee849529ad5 +size 1260306 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cb7c6fccbb088173b7ddfe494cfafca379001149 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a945bf8da73837c50240f66d862b85dfefc2c9ef4223dc5b6af7da13b3cc2a7 +size 1507207 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6251cf91ea54958b49ae4f6c13a89b070b1721ab --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f717f47f7f63a8b7782024933ba07b6251ff2c1cd576b134150c6f7e03fa672b +size 1554780 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e109051b62c8c22ff7a94e1d0a12321222e46827 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cb31531142c32910aad715f3f80bc83b51b20e8f8ba2350371fc71e36b656c4 +size 1617287 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67098eefdbe36eae0c3f5392164a79e4dc8e2f0a --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7379e8bc33b5c38d196c92d0056b519a77a0a35feba4aafdac832c3cd26ade2e +size 1465950 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2f646bb84f96fde702308f149b8ae65155c625cc --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dec0060310bb185ed9e7be5a700a91ea3a1257e2f1d26935faecf75e44c0871e +size 1506659 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95d49c5f9ad469e3e0344ffd56e3a2b291399ff8 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb16d97a7d17e520d767ebbcee4818f5adbf7a703c8ec5531ce2779703c79eb1 +size 1705295 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c858cbdbdd8f94d5b3b89b922f18b67ec01e034c --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c883e9e40544a093c095acd47f766267da343a815cbe528c6506ef129772088e +size 1688371 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..46ea635bfb7944c481e0262ec56febffcab0ae7c --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ae4f8c9bd51774c0b8924187da80f36298b869aeebdb7907814ff220087833a +size 1632571 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d65eb7187572828c36ec169e4e63a4ff62df3ab0 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f136ddfa6a75f8c38319eb69e397cb0cf05992c189494f06c72a123d44e57cad +size 1169127 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ee8814db7459a2c5f5c65b9d40795ff8ad61113d --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9019146f9223df893e271a59c4c8538311ba44dc2fd24a72c9dd4b2c64812680 +size 1170383 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9bb033310d0963d3d393c813ecc100f378b79c9d --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9352300f4a1f2d4fc59db4120a6eeb7a174fd0b71713a6348112847f63b66a3 +size 1198800 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..37fbd02ec40e4f00d8406cf967974c0c31065d8e --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff051e1686bf761b709a2c0ca3d274928506972c8da93aa7332f58ee411cb53a +size 1236656 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c757dbfc09bf24ecb7c26691a010e1afb7348b19 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a0fa497056b618ea0c00d6fc66b9c6ec244aca7406d79762d07999bb1f21f05 +size 1158233 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ef0b304c33ea4347ec5779387dd30bd13cc7d0e8 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a90b503bf7614a8b51c57e646cac1e6c87e69c6f37fa340d991743bbf1419c98 +size 1334222 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..87b5a708b3f6220669f8c15571f717c933fdd35f --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9b2fef5e246e9ba0a8b58badafa48d2ec104d3ee52c60f44f6483987e7d9ea6 +size 1195242 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0f71357a0c413878b9781be35965a7fe986780e8 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4562dd864115606b629194b8fd32ec23a69600d8aa0c75918b7b283689ef65e7 +size 1360171 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1f8e5af7b951dfdb2cd84a485a0e1860c144f89e --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cb54919d1152aace357fcead199b255676220f42ac2229e103ed67f6e84045c +size 1409917 diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8420b00ec281fdc7db7809c5d4b47018b29cd4f0 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log @@ -0,0 +1,1007 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737252228.880852]: resetting environment +[INFO] [1737252228.885466]: cleared current plan +[INFO] [1737252228.886631]: moving to home +[INFO] [1737252229.888930]: reached home +[INFO] [1737252229.890308]: reset action history +[INFO] [1737252232.598620]: environment reset +Starting episode 0 +[INFO] [1737252232.600774]: resetting environment +[INFO] [1737252232.608162]: cleared current plan +[INFO] [1737252232.609736]: moving to home +[INFO] [1737252233.612427]: reached home +[INFO] [1737252233.614089]: reset action history +[INFO] [1737252236.322884]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737252249.586374]: resetting environment +[INFO] [1737252249.589319]: cleared current plan +[INFO] [1737252249.591107]: moving to home +[INFO] [1737252250.593337]: reached home +[INFO] [1737252250.596052]: reset action history +[INFO] [1737252253.304322]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_0_2025-01-18-21-04-13.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:36207/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2577383] +process[print_text-2]: started with pid [2577407] +process[rosbag_record-3]: started with pid [2577408] +started bagging! +[INFO] [1737252280.307313]: Storing episode... +[print_text-2] killing on exit[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[INFO] [1737252281.034934]: Stored episode 1. +[INFO] [1737252281.035334]: Saving video... +[INFO] [1737252281.222293]: Video saved! +Starting episode 1 +[INFO] [1737252281.241760]: resetting environment +[INFO] [1737252281.251612]: cleared current plan +[INFO] [1737252281.251844]: moving to home +[INFO] [1737252284.854938]: reached home +[INFO] [1737252284.855410]: reset action history +[INFO] [1737252287.567691]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252292.904416]: resetting environment +[INFO] [1737252292.908375]: cleared current plan +[INFO] [1737252292.908571]: moving to home +[INFO] [1737252293.909255]: reached home +[INFO] [1737252293.909697]: reset action history +[INFO] [1737252296.616733]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_1_2025-01-18-21-04-56.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:40461/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2577550] +process[print_text-5]: started with pid [2577551] +process[rosbag_record-6]: started with pid [2577552] +started bagging! +For topic gripper_width: timestamp difference is 117696124 for nearest: 1737252313968869048 - target: 1737252313851172924 at idx 234 +[INFO] [1737252324.135120]: Storing episode... +[print_text-5] killing on exit +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[INFO] [1737252324.910110]: Stored episode 2. +[INFO] [1737252324.912991]: Saving video... +[INFO] [1737252325.103235]: Video saved! +Starting episode 2 +[INFO] [1737252325.127939]: resetting environment +[INFO] [1737252325.149159]: cleared current plan +[INFO] [1737252325.149415]: moving to home +[INFO] [1737252328.752179]: reached home +[INFO] [1737252328.755589]: reset action history +[INFO] [1737252331.529813]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252333.887250]: resetting environment +[INFO] [1737252333.897632]: cleared current plan +[INFO] [1737252333.899612]: moving to home +[INFO] [1737252334.903781]: reached home +[INFO] [1737252334.908519]: reset action history +[INFO] [1737252337.636068]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_2_2025-01-18-21-05-37.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:40403/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2577694] +process[print_text-8]: started with pid [2577695] +process[rosbag_record-9]: started with pid [2577696] +started bagging! +For topic gripper_width: timestamp difference is 107672432 for nearest: 1737252349268892744 - target: 1737252349161220312 at idx 160 +[INFO] [1737252365.192740]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1737252365.998983]: Stored episode 3. +[INFO] [1737252366.000561]: Saving video... +[INFO] [1737252366.192283]: Video saved! +Starting episode 3 +[INFO] [1737252366.239093]: resetting environment +[INFO] [1737252366.273864]: cleared current plan +[INFO] [1737252366.274213]: moving to home +[INFO] [1737252369.879619]: reached home +[INFO] [1737252369.883775]: reset action history +[INFO] [1737252372.658077]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252379.498162]: resetting environment +[INFO] [1737252379.503176]: cleared current plan +[INFO] [1737252379.508927]: moving to home +[INFO] [1737252380.513507]: reached home +[INFO] [1737252380.519886]: reset action history +[INFO] [1737252383.304430]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_3_2025-01-18-21-06-23.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:37045/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2577839] +process[print_text-11]: started with pid [2577840] +process[rosbag_record-12]: started with pid [2577841] +started bagging! +For topic gripper_width: timestamp difference is 100054451 for nearest: 1737252388168842503 - target: 1737252388068788052 at idx 59 +For topic gripper_width: timestamp difference is 102117536 for nearest: 1737252389168853042 - target: 1737252389066735506 at idx 74 +For topic gripper_width: timestamp difference is 102852856 for nearest: 1737252389502248322 - target: 1737252389399395466 at idx 77 +For topic gripper_width: timestamp difference is 113184299 for nearest: 1737252396168871250 - target: 1737252396055686951 at idx 169 +[INFO] [1737252411.329480]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1737252412.169316]: Stored episode 4. +[INFO] [1737252412.181235]: Saving video... +[INFO] [1737252412.381757]: Video saved! +Starting episode 4 +[INFO] [1737252412.433242]: resetting environment +[INFO] [1737252412.450517]: cleared current plan +[INFO] [1737252412.452329]: moving to home +[INFO] [1737252416.068716]: reached home +[INFO] [1737252416.074502]: reset action history +[INFO] [1737252418.870374]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252427.058317]: resetting environment +[INFO] [1737252427.069945]: cleared current plan +[INFO] [1737252427.071523]: moving to home +[INFO] [1737252428.073721]: reached home +[INFO] [1737252428.077697]: reset action history +[INFO] [1737252430.876560]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_4_2025-01-18-21-07-10.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:44809/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2577970] +process[print_text-14]: started with pid [2577971] +process[rosbag_record-15]: started with pid [2577972] +started bagging! +[INFO] [1737252459.983732]: Storing episode... +[rosbag_record-15] killing on exit[print_text-14] killing on exit +[pub_text-13] killing on exit +[WARN] [1737252461.117915]: Plan exhausted +[INFO] [1737252460.796740]: Stored episode 5. +[INFO] [1737252460.809417]: Saving video... +[INFO] [1737252461.038483]: Video saved! +Starting episode 5 +[INFO] [1737252461.113575]: resetting environment +[INFO] [1737252461.133040]: cleared current plan +[INFO] [1737252461.133248]: moving to home +[INFO] [1737252464.735980]: reached home +[INFO] [1737252464.745082]: reset action history +[INFO] [1737252467.610386]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252471.828540]: resetting environment +[INFO] [1737252471.856761]: cleared current plan +[INFO] [1737252471.862776]: moving to home +[INFO] [1737252472.873915]: reached home +[INFO] [1737252472.882798]: reset action history +[INFO] [1737252475.735934]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_5_2025-01-18-21-07-55.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:37537/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-16]: started with pid [2578117] +process[print_text-17]: started with pid [2578118] +process[rosbag_record-18]: started with pid [2578119] +started bagging! +[INFO] [1737252502.962404]: Storing episode... +[rosbag_record-18] killing on exit +[print_text-17] killing on exit +[pub_text-16] killing on exit +[INFO] [1737252503.827158]: Stored episode 6. +[INFO] [1737252503.831022]: Saving video... +[INFO] [1737252504.049191]: Video saved! +Starting episode 6 +[INFO] [1737252504.128759]: resetting environment +[INFO] [1737252504.149866]: cleared current plan +[INFO] [1737252504.156885]: moving to home +[INFO] [1737252507.768352]: reached home +[INFO] [1737252507.784670]: reset action history +[INFO] [1737252510.663254]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252513.556981]: resetting environment +[INFO] [1737252513.589016]: cleared current plan +[INFO] [1737252513.605209]: moving to home +[INFO] [1737252514.625871]: reached home +[INFO] [1737252514.630658]: reset action history +[INFO] [1737252517.543335]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_6_2025-01-18-21-08-37.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:32943/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-19]: started with pid [2578261] +process[print_text-20]: started with pid [2578262] +process[rosbag_record-21]: started with pid [2578263] +started bagging! +[INFO] [1737252549.414495]: Storing episode... +[rosbag_record-21] killing on exit +[print_text-20] killing on exit +[pub_text-19] killing on exit +[INFO] [1737252550.184000]: Stored episode 7. +[INFO] [1737252550.186327]: Saving video... +[INFO] [1737252550.399846]: Video saved! +Starting episode 7 +[INFO] [1737252550.493449]: resetting environment +[INFO] [1737252550.525712]: cleared current plan +[INFO] [1737252550.533201]: moving to home +[INFO] [1737252554.146813]: reached home +[INFO] [1737252554.166841]: reset action history +[INFO] [1737252557.025628]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252561.894930]: resetting environment +[INFO] [1737252561.926678]: cleared current plan +[INFO] [1737252561.928515]: moving to home +[INFO] [1737252562.937209]: reached home +[INFO] [1737252562.946625]: reset action history +[INFO] [1737252565.826750]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_7_2025-01-18-21-09-25.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:38859/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-22]: started with pid [2578397] +process[print_text-23]: started with pid [2578398] +process[rosbag_record-24]: started with pid [2578399] +started bagging! +[INFO] [1737252594.027665]: Storing episode... +[pub_text-22] killing on exit +[rosbag_record-24] killing on exit +[print_text-23] killing on exit +[INFO] [1737252594.862904]: Stored episode 8. +[INFO] [1737252594.876465]: Saving video... +[INFO] [1737252595.071871]: Video saved! +Starting episode 8 +[INFO] [1737252595.198609]: resetting environment +[INFO] [1737252595.251905]: cleared current plan +[INFO] [1737252595.258034]: moving to home +[INFO] [1737252598.875960]: reached home +[INFO] [1737252598.880645]: reset action history +[INFO] [1737252601.793511]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252604.180488]: resetting environment +[INFO] [1737252604.244191]: cleared current plan +[INFO] [1737252604.255163]: moving to home +[INFO] [1737252605.262896]: reached home +[INFO] [1737252605.274531]: reset action history +[INFO] [1737252608.276952]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_8_2025-01-18-21-10-08.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:33387/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-25]: started with pid [2578539] +process[print_text-26]: started with pid [2578540] +process[rosbag_record-27]: started with pid [2578541] +started bagging! +For topic gripper_width: timestamp difference is -109096697 for nearest: 1737252615302223989 - target: 1737252615411320686 at idx 104 +For topic gripper_width: timestamp difference is -107868905 for nearest: 1737252615902225022 - target: 1737252616010093927 at idx 108 +For topic gripper_width: timestamp difference is -102621224 for nearest: 1737252618502223108 - target: 1737252618604844332 at idx 138 +For topic gripper_width: timestamp difference is -102446817 for nearest: 1737252625702201344 - target: 1737252625804648161 at idx 227 +For topic gripper_width: timestamp difference is -100041813 for nearest: 1737252628702227646 - target: 1737252628802269459 at idx 249 +For topic gripper_width: timestamp difference is -109370277 for nearest: 1737252632302224352 - target: 1737252632411594629 at idx 249 +For topic gripper_width: timestamp difference is -106520345 for nearest: 1737252633702211449 - target: 1737252633808731794 at idx 249 +For topic gripper_width: timestamp difference is 106992533 for nearest: 1737252640468915274 - target: 1737252640361922741 at idx 249 +[INFO] [1737252640.678277]: Storing episode... +[rosbag_record-27] killing on exit +[print_text-26] killing on exit +[pub_text-25] killing on exit +[INFO] [1737252641.487795]: Stored episode 9. +[INFO] [1737252641.501824]: Saving video... +[INFO] [1737252641.717586]: Video saved! +Starting episode 9 +[INFO] [1737252641.859946]: resetting environment +[INFO] [1737252641.895232]: cleared current plan +[INFO] [1737252641.898076]: moving to home +[INFO] [1737252645.512873]: reached home +[INFO] [1737252645.515250]: reset action history +[INFO] [1737252648.490076]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252648.804707]: resetting environment +[INFO] [1737252648.820085]: cleared current plan +[INFO] [1737252648.822039]: moving to home +[INFO] [1737252649.823834]: reached home +[INFO] [1737252649.838082]: reset action history +[INFO] [1737252652.830583]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_9_2025-01-18-21-10-52.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:45817/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-28]: started with pid [2578672] +process[print_text-29]: started with pid [2578673] +process[rosbag_record-30]: started with pid [2578674] +started bagging! +For topic gripper_width: timestamp difference is -113296383 for nearest: 1737252663602251417 - target: 1737252663715547800 at idx 161 +For topic gripper_width: timestamp difference is -110479851 for nearest: 1737252665002182464 - target: 1737252665112662315 at idx 177 +For topic gripper_width: timestamp difference is -208346257 for nearest: 1737252665902246108 - target: 1737252666110592365 at idx 187 +For topic gripper_width: timestamp difference is 108212899 for nearest: 1737252680402226161 - target: 1737252680294013262 at idx 249 +[INFO] [1737252686.569738]: Storing episode... +[print_text-29] killing on exit +[rosbag_record-30] killing on exit[pub_text-28] killing on exit +[INFO] [1737252687.451693]: Stored episode 10. +[INFO] [1737252687.475513]: Saving video... +[INFO] [1737252687.700704]: Video saved! +Starting episode 10 +[INFO] [1737252687.816512]: resetting environment +[INFO] [1737252687.838705]: cleared current plan +[INFO] [1737252687.845748]: moving to home +[INFO] [1737252691.476892]: reached home +[INFO] [1737252691.493299]: reset action history +[INFO] [1737252694.558595]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252696.848747]: resetting environment +[INFO] [1737252696.874800]: cleared current plan +[INFO] [1737252696.877028]: moving to home +[INFO] [1737252697.879524]: reached home +[INFO] [1737252697.881852]: reset action history +[INFO] [1737252700.831697]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_10_2025-01-18-21-11-40.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:44435/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-31]: started with pid [2578815] +process[print_text-32]: started with pid [2578816] +process[rosbag_record-33]: started with pid [2578817] +started bagging! +[INFO] [1737252729.127484]: Storing episode... +[rosbag_record-33] killing on exit +[pub_text-31] killing on exit +[print_text-32] killing on exit +[INFO] [1737252729.946648]: Stored episode 11. +[INFO] [1737252729.954568]: Saving video... +[INFO] [1737252730.156345]: Video saved! +Starting episode 11 +[INFO] [1737252730.289805]: resetting environment +[INFO] [1737252730.332156]: cleared current plan +[INFO] [1737252730.339125]: moving to home +[INFO] [1737252733.947850]: reached home +[INFO] [1737252733.988432]: reset action history +[INFO] [1737252737.054589]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252742.437153]: resetting environment +[INFO] [1737252742.493746]: cleared current plan +[INFO] [1737252742.506896]: moving to home +[INFO] [1737252743.554389]: reached home +[INFO] [1737252743.569586]: reset action history +[INFO] [1737252746.658605]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_11_2025-01-18-21-12-26.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:33009/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-34]: started with pid [2578948] +process[print_text-35]: started with pid [2578949] +process[rosbag_record-36]: started with pid [2578950] +started bagging! +For topic gripper_width: timestamp difference is 103483262 for nearest: 1737252763468891683 - target: 1737252763365408421 at idx 234 +[WARN] [1737252776.397773]: Plan exhausted +[WARN] [1737252776.435367]: Plan exhausted +[WARN] [1737252776.483602]: Plan exhausted +preempting episode +[INFO] [1737252775.217492]: Storing episode... +[rosbag_record-36] killing on exit +[print_text-35] killing on exit +[pub_text-34] killing on exit +[INFO] [1737252776.094377]: Stored episode 12. +[INFO] [1737252776.113019]: Saving video... +[INFO] [1737252776.367151]: Video saved! +Starting episode 12 +[INFO] [1737252776.463704]: resetting environment +[INFO] [1737252776.511716]: cleared current plan +[INFO] [1737252776.522738]: moving to home +[INFO] [1737252780.152586]: reached home +[INFO] [1737252780.170799]: reset action history +[INFO] [1737252783.334437]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252784.329520]: resetting environment +[INFO] [1737252784.393610]: cleared current plan +[INFO] [1737252784.400462]: moving to home +[INFO] [1737252785.435411]: reached home +[INFO] [1737252785.464382]: reset action history +[INFO] [1737252788.525276]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_12_2025-01-18-21-13-08.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:42641/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-37]: started with pid [2579080] +process[print_text-38]: started with pid [2579081] +process[rosbag_record-39]: started with pid [2579082] +started bagging! +[INFO] [1737252824.235840]: Storing episode... +[rosbag_record-39] killing on exit +[print_text-38] killing on exit +[pub_text-37] killing on exit +[INFO] [1737252825.062088]: Stored episode 13. +[INFO] [1737252825.079984]: Saving video... +[INFO] [1737252825.307379]: Video saved! +Starting episode 13 +[INFO] [1737252825.447138]: resetting environment +[INFO] [1737252825.495707]: cleared current plan +[INFO] [1737252825.500232]: moving to home +[INFO] [1737252828.729954]: reached home +[INFO] [1737252828.754951]: reset action history +[INFO] [1737252831.859537]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252843.648989]: resetting environment +[INFO] [1737252843.709625]: cleared current plan +[INFO] [1737252843.730746]: moving to home +[INFO] [1737252844.754958]: reached home +[INFO] [1737252844.787138]: reset action history +[INFO] [1737252847.779876]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_13_2025-01-18-21-14-07.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:38463/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-40]: started with pid [2579229] +process[print_text-41]: started with pid [2579230] +process[rosbag_record-42]: started with pid [2579231] +started bagging! +For topic gripper_width: timestamp difference is -113761962 for nearest: 1737252859302221238 - target: 1737252859415983200 at idx 156 +[INFO] [1737252884.864758]: Storing episode... +[rosbag_record-42] killing on exit +[print_text-41] killing on exit +[pub_text-40] killing on exit +[WARN] [1737252885.996811]: Plan exhausted +[WARN] [1737252886.051527]: Plan exhausted +[WARN] [1737252886.075118]: Plan exhausted +[WARN] [1737252886.116323]: Plan exhausted +[WARN] [1737252886.156869]: Plan exhausted +[WARN] [1737252886.196247]: Plan exhausted +[INFO] [1737252885.751201]: Stored episode 14. +[INFO] [1737252885.756772]: Saving video... +[INFO] [1737252885.984238]: Video saved! +Starting episode 14 +[INFO] [1737252886.164837]: resetting environment +[INFO] [1737252886.202942]: cleared current plan +[INFO] [1737252886.205740]: moving to home +[INFO] [1737252889.842671]: reached home +[INFO] [1737252889.851582]: reset action history +[INFO] [1737252892.962905]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252913.230131]: resetting environment +[INFO] [1737252913.299042]: cleared current plan +[INFO] [1737252913.322768]: moving to home +[INFO] [1737252914.380588]: reached home +[INFO] [1737252914.395813]: reset action history +[INFO] [1737252917.522344]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_14_2025-01-18-21-15-17.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:41543/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-43]: started with pid [2579386] +process[print_text-44]: started with pid [2579387] +process[rosbag_record-45]: started with pid [2579388] +started bagging! +[INFO] [1737252956.072105]: Storing episode... +[print_text-44] killing on exit +[rosbag_record-45] killing on exit +[pub_text-43] killing on exit +[INFO] [1737252956.895659]: Stored episode 15. +[INFO] [1737252956.924087]: Saving video... +[INFO] [1737252957.194932]: Video saved! +[WARN] [1737252957.436092]: Plan exhausted +Starting episode 15 +[INFO] [1737252957.394524]: resetting environment +[INFO] [1737252957.444847]: cleared current plan +[INFO] [1737252957.447200]: moving to home +[INFO] [1737252960.960241]: reached home +[INFO] [1737252960.975306]: reset action history +[INFO] [1737252964.123052]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252967.227740]: resetting environment +[INFO] [1737252967.344264]: cleared current plan +[INFO] [1737252967.350844]: moving to home +[INFO] [1737252968.357203]: reached home +[INFO] [1737252968.387749]: reset action history +[INFO] [1737252971.645846]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_15_2025-01-18-21-16-11.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:43139/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-46]: started with pid [2579520] +process[print_text-47]: started with pid [2579521] +process[rosbag_record-48]: started with pid [2579522] +started bagging! +[WARN] [1737253007.157841]: Plan exhausted +[INFO] [1737253005.821412]: Storing episode... +[rosbag_record-48] killing on exit +[print_text-47] killing on exit +[pub_text-46] killing on exit +[INFO] [1737253006.683172]: Stored episode 16. +[INFO] [1737253006.723794]: Saving video... +[INFO] [1737253006.948859]: Video saved! +Starting episode 16 +[INFO] [1737253007.154919]: resetting environment +[INFO] [1737253007.185884]: cleared current plan +[INFO] [1737253007.194221]: moving to home +[INFO] [1737253010.845025]: reached home +[INFO] [1737253010.871920]: reset action history +[INFO] [1737253014.215632]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737253023.335528]: resetting environment +[INFO] [1737253023.450549]: cleared current plan +[INFO] [1737253023.466708]: moving to home +[INFO] [1737253024.496562]: reached home +[INFO] [1737253024.536683]: reset action history +[INFO] [1737253027.839437]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_16_2025-01-18-21-17-07.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:37165/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-49]: started with pid [2579665] +process[print_text-50]: started with pid [2579666] +process[rosbag_record-51]: started with pid [2579667] +started bagging! +For topic gripper_width: timestamp difference is 100826329 for nearest: 1737253043102221555 - target: 1737253043001395226 at idx 206 +For topic gripper_width: timestamp difference is 101619889 for nearest: 1737253044102224995 - target: 1737253044000605106 at idx 217 +For topic gripper_width: timestamp difference is 103516628 for nearest: 1737253046502219488 - target: 1737253046398702860 at idx 249 +[INFO] [1737253064.686105]: Storing episode... +[rosbag_record-51] killing on exit[pub_text-49] killing on exit +[print_text-50] killing on exit +[WARN] [1737253066.050252]: Plan exhausted +[WARN] [1737253066.080170]: Plan exhausted +[INFO] [1737253065.580404]: Stored episode 17. +[INFO] [1737253065.608585]: Saving video... +[INFO] [1737253065.828281]: Video saved! +Starting episode 17 +[INFO] [1737253066.028488]: resetting environment +[INFO] [1737253066.086929]: cleared current plan +[INFO] [1737253066.088937]: moving to home +[INFO] [1737253069.721228]: reached home +[INFO] [1737253069.753484]: reset action history +[INFO] [1737253073.173222]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737253080.569481]: resetting environment +[INFO] [1737253080.623943]: cleared current plan +[INFO] [1737253080.624844]: moving to home +[INFO] [1737253081.635898]: reached home +[INFO] [1737253081.651631]: reset action history +[INFO] [1737253084.858423]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_17_2025-01-18-21-18-04.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:41155/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-52]: started with pid [2579812] +process[print_text-53]: started with pid [2579813] +process[rosbag_record-54]: started with pid [2579814] +started bagging! +[INFO] [1737253126.337095]: Storing episode... +[pub_text-52] killing on exit +[rosbag_record-54] killing on exit +[print_text-53] killing on exit +[INFO] [1737253127.156114]: Stored episode 18. +[INFO] [1737253127.162560]: Saving video... +[INFO] [1737253127.418363]: Video saved! +Starting episode 18 +[INFO] [1737253127.618836]: resetting environment +[INFO] [1737253127.666623]: cleared current plan +[INFO] [1737253127.679498]: moving to home +[INFO] [1737253131.319667]: reached home +[INFO] [1737253131.349984]: reset action history +[INFO] [1737253134.558201]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737253146.904846]: resetting environment +[INFO] [1737253146.994045]: cleared current plan +[INFO] [1737253147.002173]: moving to home +[INFO] [1737253148.016361]: reached home +[INFO] [1737253148.042839]: reset action history +[INFO] [1737253151.428010]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_18_2025-01-18-21-19-11.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:45119/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-55]: started with pid [2579969] +process[print_text-56]: started with pid [2579970] +process[rosbag_record-57]: started with pid [2579971] +started bagging! +[WARN] [1737253193.040826]: Plan exhausted +[WARN] [1737253193.082294]: Plan exhausted +[WARN] [1737253193.152706]: Plan exhausted +[WARN] [1737253193.195307]: Plan exhausted +[WARN] [1737253193.212877]: Plan exhausted +[INFO] [1737253191.703180]: Storing episode... +[rosbag_record-57] killing on exit +[pub_text-55] killing on exit[print_text-56] killing on exit +[INFO] [1737253192.540248]: Stored episode 19. +[INFO] [1737253192.549114]: Saving video... +[INFO] [1737253192.842015]: Video saved! +Starting episode 19 +[INFO] [1737253193.190228]: resetting environment +[INFO] [1737253193.230828]: cleared current plan +[INFO] [1737253193.235833]: moving to home +[INFO] [1737253196.865658]: reached home +[INFO] [1737253196.877187]: reset action history +[INFO] [1737253200.137476]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737253207.808285]: resetting environment +[INFO] [1737253207.911884]: cleared current plan +[INFO] [1737253207.921422]: moving to home +[INFO] [1737253208.942451]: reached home +[INFO] [1737253208.979051]: reset action history +[INFO] [1737253212.462321]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_19_2025-01-18-21-20-12.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log +started roslaunch server http://158.130.50.37:34221/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-58]: started with pid [2580114] +process[print_text-59]: started with pid [2580115] +process[rosbag_record-60]: started with pid [2580116] +started bagging! +For topic gripper_width: timestamp difference is -244909480 for nearest: 1737253240902248189 - target: 1737253241147157669 at idx 249 +[INFO] [1737253252.627879]: Storing episode... +[print_text-59] killing on exit +[rosbag_record-60] killing on exit +[pub_text-58] killing on exit +[INFO] [1737253253.520209]: Stored episode 20. +[INFO] [1737253253.544701]: Saving video... +[WARN] [1737253254.006426]: Plan exhausted +[INFO] [1737253253.801915]: Video saved! +[INFO] [1737253253.988822]: resetting environment +[INFO] [1737253254.025843]: cleared current plan +[INFO] [1737253254.026109]: moving to home +[INFO] [1737253257.643344]: reached home +[INFO] [1737253257.654607]: reset action history +[INFO] [1737253260.920305]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/requirements.txt b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch new file mode 100644 index 0000000000000000000000000000000000000000..3ac2f303b7f4f5f22e13197da20e546ff1801dc0 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch @@ -0,0 +1,119 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..c11eb32 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 5 ++num_eval: 20 + + debug_timestamps: False + open_loop: False +@@ -132,15 +132,25 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++# wandb_run_id: '3456_2' # seed 2 ++ ++# all books, 06/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3465_0' ++# wandb_run_id: '3465_1' # seed 1 ++wandb_run_id: '3465_2' # seed 2 ++ + # RGBD+mask+act history +-# wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history +@@ -191,7 +201,7 @@ wandb_run_id: '1017_0' # dataset shuffle seed 1 + true_action_history: false + wandb_notes: null + +-checkpoint_epoch: 12000 # for depth + all contact ++checkpoint_epoch: 7500 # for depth + all contact + load_residual_weight: false + + checkpoint_root_dir: '/home/${oc.env:USER}/fish_leon/FISH' +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..9509617 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,15 +167,21 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ # '3456_1', ++ # '3456_2', ++ '3465_0', ++ '3465_1', ++ '3465_2', + ] + +-checkpoint_epoch = 12000 ++checkpoint_epoch = 7500 + + # use subprocess to download the model checkpoints in parallel + process_list = [] +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-metadata.json b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c328763ac4bdd71d4edb870ca6042502428defd6 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-19T02:03:44.567264", + "startedAt": "2025-01-19T02:03:44.029624", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "ec42957dc10d08dbd8b134558fcd8f9e470ccc31" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3558.444375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4480.37, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4488.089, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4562.354, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 658.0720520019531 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f8238f69f6cadb163e1045f76882fbfbbdc246ed --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "373935c9a53d50a21516ae9864a2b682a53224d448bd3a5444b7c73d070560df", "size": 1151995, "path": "media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4"}, "global_step": 2618, "_timestamp": 1737253282.9134135, "_runtime": 1058.8758735656738, "_step": 41, "eval/1_eval": {"_type": "video-file", "sha256": "f136ddfa6a75f8c38319eb69e397cb0cf05992c189494f06c72a123d44e57cad", "size": 1169127, "path": "media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4"}, "eval/num_success": 15.0, "episode": 20.0, "eval/success_rate": 0.75, "eval/success_rate_lb_0.01": 0.4808793067932129, "eval/success_rate_ub_0.01": 0.9195476174354553, "eval/success_rate_lb_0.025": 0.5107601881027222, "eval/success_rate_ub_0.025": 0.9122562408447266, "eval/success_rate_lb_0.05": 0.5460197925567627, "eval/success_rate_ub_0.05": 0.8860257863998413, "eval/success_rate_lb_0.1": 0.6174886226654053, "eval/success_rate_ub_0.1": 0.8676386475563049, "eval/2_eval": {"_type": "video-file", "sha256": "9019146f9223df893e271a59c4c8538311ba44dc2fd24a72c9dd4b2c64812680", "size": 1170383, "path": "media/videos/eval/2_eval_2_9019146f9223df893e27.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "a9352300f4a1f2d4fc59db4120a6eeb7a174fd0b71713a6348112847f63b66a3", "size": 1198800, "path": "media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "ff051e1686bf761b709a2c0ca3d274928506972c8da93aa7332f58ee411cb53a", "size": 1236656, "path": "media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "1a0fa497056b618ea0c00d6fc66b9c6ec244aca7406d79762d07999bb1f21f05", "size": 1158233, "path": "media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "a90b503bf7614a8b51c57e646cac1e6c87e69c6f37fa340d991743bbf1419c98", "size": 1334222, "path": "media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "d9b2fef5e246e9ba0a8b58badafa48d2ec104d3ee52c60f44f6483987e7d9ea6", "size": 1195242, "path": "media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "4562dd864115606b629194b8fd32ec23a69600d8aa0c75918b7b283689ef65e7", "size": 1360171, "path": "media/videos/eval/8_eval_8_4562dd864115606b6291.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "0cb54919d1152aace357fcead199b255676220f42ac2229e103ed67f6e84045c", "size": 1409917, "path": "media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "194dcd6049c4233da56c1320d7fff1f54a4b9ebb30a13b10b4abae786a908461", "size": 1191627, "path": "media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "136a95e750557f39dd091db067809586d46c1938c60d2ff38e1ecee849529ad5", "size": 1260306, "path": "media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "8a945bf8da73837c50240f66d862b85dfefc2c9ef4223dc5b6af7da13b3cc2a7", "size": 1507207, "path": "media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "f717f47f7f63a8b7782024933ba07b6251ff2c1cd576b134150c6f7e03fa672b", "size": 1554780, "path": "media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "6cb31531142c32910aad715f3f80bc83b51b20e8f8ba2350371fc71e36b656c4", "size": 1617287, "path": "media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4"}, "eval/15_eval": {"_type": "video-file", "sha256": "7379e8bc33b5c38d196c92d0056b519a77a0a35feba4aafdac832c3cd26ade2e", "size": 1465950, "path": "media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4"}, "eval/16_eval": {"_type": "video-file", "sha256": "dec0060310bb185ed9e7be5a700a91ea3a1257e2f1d26935faecf75e44c0871e", "size": 1506659, "path": "media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4"}, "eval/17_eval": {"_type": "video-file", "sha256": "cb16d97a7d17e520d767ebbcee4818f5adbf7a703c8ec5531ce2779703c79eb1", "size": 1705295, "path": "media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4"}, "eval/18_eval": {"_type": "video-file", "sha256": "c883e9e40544a093c095acd47f766267da343a815cbe528c6506ef129772088e", "size": 1688371, "path": "media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4"}, "eval/19_eval": {"_type": "video-file", "sha256": "0ae4f8c9bd51774c0b8924187da80f36298b869aeebdb7907814ff220087833a", "size": 1632571, "path": "media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "046a987f9352685cc2d759c41835118b88947e1e3df078eb7a4f4ffebb465fc5", "size": 398, "artifact_path": "wandb-client-artifact://vv30ligbro5fyesov32qmtodg4xsdj71zbv0ody8s8mqax367k7mivhlxlt1ab9tgt8xlcdvil6esouwz82mcn94m7npek0dolxn26bh7b8dnn5xtv3nccxgxjgl23mt/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://72ccfgzhmctyg2vi7h3ni5p4qethd2n60ebfjwb07b0ks72hfl3elsu6a6zrbzr356ogspbuae13gu79a4y0sgdvbwi2vttcvyovs0hnxxb877v40ae60dk0abn9mgud:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 15, "total_num_episodes": 20, "total_success_rate": 0.75, "total_success_rate_lb_0.01": 0.510935544965464, "total_success_rate_ub_0.01": 0.9235437958585514, "total_success_rate_lb_0.025": 0.5362904965804953, "total_success_rate_ub_0.025": 0.9076186566729699, "total_success_rate_lb_0.05": 0.5726168081019127, "total_success_rate_ub_0.05": 0.8630436881650583, "total_success_rate_lb_0.1": 0.6052489131478573, "total_success_rate_ub_0.1": 0.8673701099316663, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "984e345eb950dc10e24c64e129e0a95bff97f4e90aecafb473b83ce1523bdc29", "size": 505, "artifact_path": "wandb-client-artifact://377tix1vmm7bawpeny882t8l5tb7k87vtl3knusx2z5xldsxa5fr27wkza28dbudxpkqjeek0ceekmidzox3wwwg1uqkl35cgyebxucclsu7i8eiy9uqfzekinggpkzj/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://xyjcb4oz9cwqpjoor8g9vpgr8rnmwf507g1zahlti4dz8rdf508lkl4btqhqtvsoxa2vzldjm4ttpk3djygjwqxwhmzrr06pokzm3d3p0h2oykxuw7shpsl5c0dj2ed4:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 1065}, "eval/total_success_rate": 0.75, "eval/total_success_rate_lb_0.01": 0.5109355449676514, "eval/total_success_rate_ub_0.01": 0.9235438108444214, "eval/total_success_rate_lb_0.025": 0.5362904667854309, "eval/total_success_rate_ub_0.025": 0.9076186418533325, "eval/total_success_rate_lb_0.05": 0.5726168155670166, "eval/total_success_rate_ub_0.05": 0.8630436658859253, "eval/total_success_rate_lb_0.1": 0.6052489280700684, "eval/total_success_rate_ub_0.1": 0.8673701286315918} \ No newline at end of file diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/logs/debug-internal.log b/2t6elatx/wandb/run-20250118_210344-2t6elatx/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9db61d47fbd543c4968821d00ad8f3ba31fe4b28 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/logs/debug-internal.log @@ -0,0 +1,2254 @@ +2025-01-18 21:03:44,037 INFO StreamThr :2577265 [internal.py:wandb_internal():85] W&B internal server running at pid: 2577265, started at: 2025-01-18 21:03:44.037545 +2025-01-18 21:03:44,039 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status +2025-01-18 21:03:44,053 INFO WriterThread:2577265 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/run-2t6elatx.wandb +2025-01-18 21:03:44,054 DEBUG SenderThread:2577265 [sender.py:send():379] send: header +2025-01-18 21:03:44,061 DEBUG SenderThread:2577265 [sender.py:send():379] send: run +2025-01-18 21:03:44,485 INFO SenderThread:2577265 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files +2025-01-18 21:03:44,485 INFO SenderThread:2577265 [sender.py:_start_run_threads():1188] run started: 2t6elatx with start time 1737252224.03754 +2025-01-18 21:03:44,500 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: check_version +2025-01-18 21:03:44,500 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: check_version +2025-01-18 21:03:44,548 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: run_start +2025-01-18 21:03:44,553 DEBUG HandlerThread:2577265 [system_info.py:__init__():26] System info init +2025-01-18 21:03:44,553 DEBUG HandlerThread:2577265 [system_info.py:__init__():41] System info init done +2025-01-18 21:03:44,553 INFO HandlerThread:2577265 [system_monitor.py:start():194] Starting system monitor +2025-01-18 21:03:44,554 INFO SystemMonitor:2577265 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-18 21:03:44,554 INFO HandlerThread:2577265 [system_monitor.py:probe():214] Collecting system info +2025-01-18 21:03:44,554 INFO SystemMonitor:2577265 [interfaces.py:start():188] Started cpu monitoring +2025-01-18 21:03:44,555 INFO SystemMonitor:2577265 [interfaces.py:start():188] Started disk monitoring +2025-01-18 21:03:44,555 INFO SystemMonitor:2577265 [interfaces.py:start():188] Started gpu monitoring +2025-01-18 21:03:44,556 INFO SystemMonitor:2577265 [interfaces.py:start():188] Started memory monitoring +2025-01-18 21:03:44,557 INFO SystemMonitor:2577265 [interfaces.py:start():188] Started network monitoring +2025-01-18 21:03:44,567 DEBUG HandlerThread:2577265 [system_info.py:probe():152] Probing system +2025-01-18 21:03:44,568 DEBUG HandlerThread:2577265 [system_info.py:_probe_git():137] Probing git +2025-01-18 21:03:44,573 DEBUG HandlerThread:2577265 [system_info.py:_probe_git():145] Probing git done +2025-01-18 21:03:44,573 DEBUG HandlerThread:2577265 [system_info.py:probe():200] Probing system done +2025-01-18 21:03:44,573 DEBUG HandlerThread:2577265 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-19T02:03:44.567264', 'startedAt': '2025-01-19T02:03:44.029624', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3558.444375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4480.37, 'min': 800.0, 'max': 4700.0}, {'current': 4488.089, 'min': 800.0, 'max': 4700.0}, {'current': 4562.354, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 658.0720520019531}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-18 21:03:44,573 INFO HandlerThread:2577265 [system_monitor.py:probe():224] Finished collecting system info +2025-01-18 21:03:44,573 INFO HandlerThread:2577265 [system_monitor.py:probe():227] Publishing system info +2025-01-18 21:03:44,573 DEBUG HandlerThread:2577265 [system_info.py:_save_code():44] Saving code +2025-01-18 21:03:44,574 DEBUG HandlerThread:2577265 [system_info.py:_save_code():67] Saving code done +2025-01-18 21:03:44,575 DEBUG HandlerThread:2577265 [system_info.py:_save_patches():84] Saving git patches +2025-01-18 21:03:44,669 DEBUG HandlerThread:2577265 [system_info.py:_save_patches():126] Saving git patches done +2025-01-18 21:03:44,670 INFO HandlerThread:2577265 [system_monitor.py:probe():229] Finished publishing system info +2025-01-18 21:03:44,673 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:03:44,673 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-18 21:03:44,673 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-18 21:03:44,673 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-18 21:03:44,674 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-18 21:03:44,904 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: python_packages +2025-01-18 21:03:44,904 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:03:44,904 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: python_packages +2025-01-18 21:03:44,905 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:03:44,906 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:44,906 INFO HandlerThread:2577265 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/tb" + save: true +} +control { + relay_id: "bcf15667ba81445d81be1061d1a1388d" +} +_info { + stream_id: "2t6elatx" +} + +2025-01-18 21:03:44,907 DEBUG HandlerThread:2577265 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-18 21:03:44,931 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/a4oe7r2l-wandb-metadata.json +2025-01-18 21:03:44,938 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/cl0olw9t-code/FISH/eval_robot.py +2025-01-18 21:03:44,946 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/h5brxswo-diff.patch +2025-01-18 21:03:44,949 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/f3we0mxa-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 21:03:45,019 DEBUG SenderThread:2577265 [sender.py:send():379] send: telemetry +2025-01-18 21:03:45,019 DEBUG SenderThread:2577265 [sender.py:send():379] send: tbrecord +2025-01-18 21:03:45,020 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:03:45,020 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 with policy live +2025-01-18 21:03:45,201 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/vu00izql-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:03:45,487 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/requirements.txt +2025-01-18 21:03:45,487 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:03:45,487 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/code/FISH/eval_robot.py +2025-01-18 21:03:45,487 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 21:03:45,487 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/diff.patch +2025-01-18 21:03:45,488 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:03:45,488 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-metadata.json +2025-01-18 21:03:45,488 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/code/FISH +2025-01-18 21:03:45,488 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/code +2025-01-18 21:03:45,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:46,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:47,487 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:03:47,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:48,880 DEBUG SenderThread:2577265 [sender.py:send():379] send: config +2025-01-18 21:03:48,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:49,889 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:03:49,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:50,906 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:51,488 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:03:51,906 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:52,906 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:53,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:54,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:03:54,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:55,489 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:03:55,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:56,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:57,489 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:03:57,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:58,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:59,904 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:03:59,904 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:03:59,945 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:00,046 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:00,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:01,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:02,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:03,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:04,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:05,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:05,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:06,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:07,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:08,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:09,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:10,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:11,493 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:11,637 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:11,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:12,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:13,494 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:13,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:14,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:04:14,905 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:04:14,945 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:15,494 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:15,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:16,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:16,986 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:17,495 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/config.yaml +2025-01-18 21:04:17,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:18,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:19,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:20,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:21,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:22,906 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:22,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:23,911 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:24,913 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:25,913 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:26,913 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:27,906 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:27,913 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:28,913 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:29,904 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:04:29,904 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:04:29,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:30,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:31,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:32,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:33,058 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:33,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:34,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:35,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:36,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:37,914 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:38,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:38,915 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:39,915 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:40,915 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:41,240 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:04:41,240 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 with policy now +2025-01-18 21:04:41,240 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:04:41,241 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:04:41,241 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:04:41,242 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:04:41,503 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:41,503 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:04:41,503 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 +2025-01-18 21:04:41,503 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media +2025-01-18 21:04:41,503 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:04:41,503 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos +2025-01-18 21:04:41,508 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/a4g7ag7y-media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 +2025-01-18 21:04:41,799 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/9nlf524s-media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 +2025-01-18 21:04:41,915 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:42,915 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:43,503 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:43,915 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:44,252 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:44,557 DEBUG SystemMonitor:2577265 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-18 21:04:44,558 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:04:44,904 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:04:44,905 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:04:44,916 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:45,917 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:46,916 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:47,505 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:47,916 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:48,917 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:49,505 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:49,609 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:49,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:50,917 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:51,917 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:52,937 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:53,507 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:04:53,768 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/i4lnxezk-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:04:53,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:54,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:04:54,917 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:55,507 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:55,918 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:56,918 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:57,918 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:58,918 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:04:59,509 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:04:59,904 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:04:59,905 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:04:59,945 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:00,029 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:00,918 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:01,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:02,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:03,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:04,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:05,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:05,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:06,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:07,919 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:08,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:09,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:10,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:10,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:11,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:12,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:13,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:14,566 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:05:14,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:05:14,905 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:05:14,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:15,514 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:15,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:15,990 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:16,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:17,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:18,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:19,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:20,922 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:21,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:21,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:22,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:23,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:24,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:25,145 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:05:25,146 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:05:25,146 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 with policy now +2025-01-18 21:05:25,146 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:05:25,146 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:05:25,146 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:05:25,416 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/u5p126k7-media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 +2025-01-18 21:05:25,517 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:25,517 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:05:25,517 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 +2025-01-18 21:05:25,517 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:05:25,822 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/jpfn1j2z-media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 +2025-01-18 21:05:25,926 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:26,925 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:27,151 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:27,518 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:27,926 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:28,927 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:29,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:05:29,910 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:05:29,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:30,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:31,519 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:31,926 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:32,573 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:32,933 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:33,520 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:33,931 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:34,313 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:05:34,313 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:05:34,313 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:05:34,520 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:05:34,520 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:05:34,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:35,521 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:35,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:36,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:37,521 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:37,636 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:37,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:38,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:39,522 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:39,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:40,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:41,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:42,911 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:42,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:43,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:44,559 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:05:44,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:05:44,905 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:05:44,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:45,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:46,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:47,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:47,993 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:48,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:49,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:50,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:51,526 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:05:51,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:52,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:53,911 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:05:53,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:54,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:55,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:56,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:57,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:58,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:59,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:05:59,905 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:05:59,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:05:59,997 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:00,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:01,931 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:02,931 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:03,932 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:04,932 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:05,193 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:05,941 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:06,234 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:06:06,234 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 with policy now +2025-01-18 21:06:06,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:06:06,237 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:06:06,237 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:06:06,238 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:06:06,531 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:06:06,531 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 +2025-01-18 21:06:06,531 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:06:06,538 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/8kjphwxb-media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 +2025-01-18 21:06:06,866 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/c5b0q6f7-media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 +2025-01-18 21:06:06,934 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:07,531 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:07,937 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:08,937 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:09,938 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:10,921 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:10,940 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:11,532 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:11,938 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:12,939 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:13,533 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:13,943 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:14,559 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:06:14,934 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:06:14,934 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:06:14,977 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:15,534 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:15,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:16,004 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:16,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:17,950 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:18,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:19,535 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:06:19,828 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/5sqph8os-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:06:19,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:20,151 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:06:20,151 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:06:20,151 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:06:20,536 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:06:20,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:21,536 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:21,557 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:21,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:22,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:23,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:24,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:25,537 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:25,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:26,922 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:26,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:27,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:28,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:29,539 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:29,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:06:29,920 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:06:29,950 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:30,950 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:31,539 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:31,950 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:32,070 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:32,950 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:33,950 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:34,950 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:35,951 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:36,951 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:37,225 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:37,542 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:37,951 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:38,951 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:39,951 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:40,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:41,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:42,922 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:42,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:43,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:44,560 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:06:44,918 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:06:44,919 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:06:44,961 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:45,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:46,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:47,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:48,006 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:48,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:49,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:50,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:51,956 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:52,430 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:06:52,430 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 with policy now +2025-01-18 21:06:52,433 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:06:52,433 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:06:52,433 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:06:52,434 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:06:52,557 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:06:52,558 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 +2025-01-18 21:06:52,558 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:06:52,694 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/6be1rgm0-media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 +2025-01-18 21:06:52,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:52,989 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/2gh50kbn-media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 +2025-01-18 21:06:53,454 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:53,558 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:53,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:54,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:55,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:56,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:57,559 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:06:57,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:58,871 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:06:58,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:06:59,925 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:06:59,925 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:06:59,965 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:00,968 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:01,560 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:01,966 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:02,975 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:03,972 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:04,065 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:04,974 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:05,973 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:06,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:07,207 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:07:07,207 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:07:07,207 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:07:07,562 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:07:07,563 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:07:07,968 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:08,978 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:09,117 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:09,563 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:09,982 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:10,974 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:11,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:12,972 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:13,565 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:13,973 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:14,284 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:14,560 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:07:14,920 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:07:14,920 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:07:14,977 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:15,565 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:15,973 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:16,973 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:17,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:18,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:19,932 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:19,975 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:20,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:21,975 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:22,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:23,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:24,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:25,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:25,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:26,978 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:27,977 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:28,977 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:29,923 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:07:29,923 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:07:29,977 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:30,977 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:31,006 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:31,977 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:32,978 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:33,978 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:34,978 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:35,978 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:36,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:36,981 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:37,979 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:38,979 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:39,980 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:40,992 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:41,097 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:07:41,097 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 with policy now +2025-01-18 21:07:41,098 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:07:41,098 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:07:41,099 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:07:41,100 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:07:41,367 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/29jfdqu6-media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 +2025-01-18 21:07:41,574 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:41,575 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:07:41,575 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 +2025-01-18 21:07:41,575 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:07:41,873 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/yjzneliy-media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 +2025-01-18 21:07:41,989 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:42,133 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:42,989 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:43,575 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:43,986 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:44,561 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:07:44,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:07:44,949 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:07:44,989 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:45,576 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:45,992 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:46,991 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:47,611 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:47,986 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:48,990 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:49,577 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:50,006 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:51,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:52,004 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:52,260 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:07:52,260 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:07:52,262 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:07:52,578 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:07:52,579 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:07:52,843 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/r19j16bi-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:07:52,876 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:53,009 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:53,579 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:54,004 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:55,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:55,579 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:56,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:56,999 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:57,580 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:07:57,999 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:58,247 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:07:58,999 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:07:59,923 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:07:59,923 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:07:59,999 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:00,999 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:01,999 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:03,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:04,000 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:04,077 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:05,000 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:06,000 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:07,000 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:08,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:09,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:09,932 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:10,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:11,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:12,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:13,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:14,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:14,561 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:08:14,923 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:08:14,923 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:08:15,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:15,012 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:16,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:17,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:18,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:19,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:20,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:20,933 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:21,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:22,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:23,005 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:24,009 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:24,117 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:08:24,117 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 with policy now +2025-01-18 21:08:24,123 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:08:24,123 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:08:24,123 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:08:24,124 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:08:24,382 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/4bra0h1v-media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 +2025-01-18 21:08:24,591 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:08:24,591 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 +2025-01-18 21:08:24,591 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:08:24,884 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/uxndxhhz-media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 +2025-01-18 21:08:25,021 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:25,591 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:08:25,941 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:26,022 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:27,023 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:28,024 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:29,020 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:29,592 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:08:29,928 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:08:29,928 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:08:30,028 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:31,013 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:31,593 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:08:31,709 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:32,026 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:33,028 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:33,594 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:08:34,026 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:34,312 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:08:34,313 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:08:34,314 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:08:34,594 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:08:34,594 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:08:35,029 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:35,595 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:08:36,030 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:36,941 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:37,041 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:38,049 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:39,042 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:39,596 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:08:40,040 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:41,041 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:42,029 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:42,513 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:43,030 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:43,597 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:08:44,030 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:44,561 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:08:44,927 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:08:44,927 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:08:45,030 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:46,030 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:47,030 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:48,031 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:48,036 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:49,031 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:50,031 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:51,031 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:52,031 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:53,031 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:53,940 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:08:54,032 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:55,032 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:56,032 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:57,032 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:58,032 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:59,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:08:59,927 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:08:59,927 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:08:59,994 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:00,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:01,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:02,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:03,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:04,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:05,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:05,940 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:06,034 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:07,034 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:08,034 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:09,034 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:10,042 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:10,477 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:09:10,478 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 with policy now +2025-01-18 21:09:10,486 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:09:10,486 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:09:10,486 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:09:10,487 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:09:10,607 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:09:10,608 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 +2025-01-18 21:09:10,608 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:09:10,751 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/2bl04fqg-media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 +2025-01-18 21:09:11,050 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:11,078 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/mxlj1ylo-media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 +2025-01-18 21:09:11,534 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:11,608 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:12,047 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:13,053 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:14,061 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:14,562 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:09:14,955 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:09:14,955 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:09:15,056 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:15,609 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:16,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:17,026 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:17,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:18,092 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:19,082 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:19,611 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:20,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:21,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:22,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:22,378 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:09:22,378 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:09:22,378 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:22,378 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:09:22,612 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:09:22,612 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:09:22,873 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/c7lpznkb-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:09:23,095 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:23,612 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:24,092 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:25,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:25,613 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:26,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:27,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:27,614 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:28,086 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:28,321 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:29,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:29,615 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:29,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:09:29,930 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:09:30,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:31,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:32,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:33,085 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:34,015 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:34,086 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:35,086 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:36,086 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:37,086 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:38,086 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:39,086 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:39,948 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:40,088 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:41,089 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:42,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:43,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:44,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:44,562 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:09:44,570 WARNING FileStreamThread:2577265 [file_stream.py:request_with_retry():674] requests_with_retry encountered retryable exception: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')). func: >, args: ('https://api.wandb.ai/files/serialexperimentsleon/extrinsic_contact_downstream/2t6elatx/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"eval/0_eval": {"_type": "video-file", "sha256": "373935c9a53d50a21516ae9864a2b682a53224d448bd3a5444b7c73d070560df", "size": 1151995, "path": "media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4"}, "global_step": 796, "_timestamp": 1737252513.5501094, "_runtime": 289.51256942749023, "_step": 12, "eval/1_eval": {"_type": "video-file", "sha256": "f136ddfa6a75f8c38319eb69e397cb0cf05992c189494f06c72a123d44e57cad", "size": 1169127, "path": "media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4"}, "eval/num_success": 6.0, "episode": 6.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.8098405599594116, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.780625581741333, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6978645324707031, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.7496918439865112, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "9019146f9223df893e271a59c4c8538311ba44dc2fd24a72c9dd4b2c64812680", "size": 1170383, "path": "media/videos/eval/2_eval_2_9019146f9223df893e27.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "a9352300f4a1f2d4fc59db4120a6eeb7a174fd0b71713a6348112847f63b66a3", "size": 1198800, "path": "media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "ff051e1686bf761b709a2c0ca3d274928506972c8da93aa7332f58ee411cb53a", "size": 1236656, "path": "media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "1a0fa497056b618ea0c00d6fc66b9c6ec244aca7406d79762d07999bb1f21f05", "size": 1158233, "path": "media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "a90b503bf7614a8b51c57e646cac1e6c87e69c6f37fa340d991743bbf1419c98", "size": 1334222, "path": "media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4"}}']}, 'wandb-history.jsonl': {'offset': 12, 'content': ['{"global_step": 796, "_timestamp": 1737252513.5501094, "eval/num_success": 6.0, "episode": 6.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.8098405599594116, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.780625581741333, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6978645324707031, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.7496918439865112, "eval/success_rate_ub_0.1": 1.0, "_runtime": 289.51256942749023, "_step": 12}']}, 'wandb-events.jsonl': {'offset': 9, 'content': ['{"system.proc.memory.availableMB": 54499.66, "system.memory": 15.13, "system.proc.memory.rssMB": 940.93, "system.proc.memory.percent": 1.47, "system.cpu": 13.47, "system.cpu.0.cpu_percent": 23.32, "system.cpu.1.cpu_percent": 23.09, "system.cpu.2.cpu_percent": 49.47, "system.cpu.3.cpu_percent": 26.25, "system.cpu.4.cpu_percent": 27.35, "system.cpu.5.cpu_percent": 24.99, "system.cpu.6.cpu_percent": 24.06, "system.cpu.7.cpu_percent": 24.03, "system.proc.cpu.threads": 59, "system.network.sent": 41607873249.27, "system.network.recv": 28071465410.33, "system.disk.\\\\.usagePercent": 76.67, "system.disk.\\\\.usageGB": 665.91, "system.disk.in": 0.27, "system.disk.out": 7344.52, "system.gpu.0.memoryAllocated": 28.37, "system.gpu.0.memoryAllocatedBytes": 2437205196.8, "system.gpu.0.memory": 4.0, "system.gpu.0.gpu": 13.13, "system.gpu.0.temp": 47.73, "system.gpu.0.powerWatts": 49.35, "system.gpu.0.powerPercent": 22.63, "_wandb": true, "_timestamp": 1737252554.561908, "_runtime": 330.524368}', '{"system.proc.memory.availableMB": 54431.22, "system.memory": 15.25, "system.proc.memory.rssMB": 933.58, "system.proc.memory.percent": 1.45, "system.cpu": 7.43, "system.cpu.0.cpu_percent": 31.45, "system.cpu.1.cpu_percent": 29.58, "system.cpu.2.cpu_percent": 47.41, "system.cpu.3.cpu_percent": 30.83, "system.cpu.4.cpu_percent": 27.8, "system.cpu.5.cpu_percent": 29.28, "system.cpu.6.cpu_percent": 26.15, "system.cpu.7.cpu_percent": 23.95, "system.proc.cpu.threads": 56, "system.network.sent": 46013138381.0, "system.network.recv": 31047672563.0, "system.disk.\\\\.usagePercent": 76.75, "system.disk.\\\\.usageGB": 666.75, "system.disk.in": 0.29, "system.disk.out": 8135.24, "system.gpu.0.memoryAllocated": 28.37, "system.gpu.0.memoryAllocatedBytes": 2436934314.67, "system.gpu.0.memory": 8.73, "system.gpu.0.gpu": 31.4, "system.gpu.0.temp": 49.8, "system.gpu.0.powerWatts": 57.08, "system.gpu.0.powerPercent": 26.55, "_wandb": true, "_timestamp": 1737252584.56238, "_runtime": 360.52484}']}, 'output.log': {'offset': 327, 'content': ['2025-01-19T02:09:11.352145 [INFO] [1737252549.414495]: Storing episode...\n', '2025-01-19T02:09:11.352145 [rosbag_record-21] killing on exit\n', '2025-01-19T02:09:11.352145 [print_text-20] killing on exit\n', '2025-01-19T02:09:11.352145 [pub_text-19] killing on exit\n', '2025-01-19T02:09:11.352145 [INFO] [1737252550.184000]: Stored episode 7.\n', '2025-01-19T02:09:11.352145 [INFO] [1737252550.186327]: Saving video...\n', '2025-01-19T02:09:11.352145 [INFO] [1737252550.399846]: Video saved!\n', '2025-01-19T02:09:11.352145 Starting episode 7\n', '2025-01-19T02:09:11.352145 [INFO] [1737252550.493449]: resetting environment\n', '2025-01-19T02:09:11.352145 [INFO] [1737252550.525712]: cleared current plan\n', '2025-01-19T02:09:11.352145 [INFO] [1737252550.533201]: moving to home\n', '2025-01-19T02:09:15.355592 [INFO] [1737252554.146813]: reached home\n', '2025-01-19T02:09:15.355592 [INFO] [1737252554.166841]: reset action history\n', '2025-01-19T02:09:19.358922 [INFO] [1737252557.025628]: environment reset\n', '2025-01-19T02:09:19.358922 Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success\n', '2025-01-19T02:09:23.364233 proceeding to start episode!\n', '2025-01-19T02:09:23.364233 Comuting lower confidence bound\n', '2025-01-19T02:09:23.364233 Comuting upper confidence bound\n', '2025-01-19T02:09:23.364233 Comuting lower confidence bound\n', '2025-01-19T02:09:23.364233 Comuting upper confidence bound\n', '2025-01-19T02:09:23.364233 Comuting lower confidence bound\n', '2025-01-19T02:09:23.364233 Comuting upper confidence bound\n', '2025-01-19T02:09:23.364233 Comuting lower confidence bound\n', '2025-01-19T02:09:23.364233 Comuting upper confidence bound\n', '2025-01-19T02:09:23.364233 [INFO] [1737252561.894930]: resetting environment\n', '2025-01-19T02:09:23.364233 [INFO] [1737252561.926678]: cleared current plan\n', '2025-01-19T02:09:23.364233 [INFO] [1737252561.928515]: moving to home\n', '2025-01-19T02:09:25.365484 [INFO] [1737252562.937209]: reached home\n', '2025-01-19T02:09:25.365484 [INFO] [1737252562.946625]: reset action history\n', '2025-01-19T02:09:27.367948 [INFO] [1737252565.826750]: environment reset\n', '2025-01-19T02:09:27.367948 bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/episode_rosbags/episode_7_2025-01-18-21-09-25.bag\n', '2025-01-19T02:09:27.367948 ... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2577081.log\n', '2025-01-19T02:09:29.370300 \x1b[1mstarted roslaunch server http://158.130.50.37:38859/\n', '2025-01-19T02:09:29.370300 SUMMARY\n', '2025-01-19T02:09:29.370300 ========\n', '2025-01-19T02:09:29.370300 PARAMETERS\n', '2025-01-19T02:09:29.370300 * /rosdistro: noetic\n', '2025-01-19T02:09:29.370300 * /rosversion: 1.16.0\n', '2025-01-19T02:09:29.370300 NODES\n', '2025-01-19T02:09:29.370300 /\n', '2025-01-19T02:09:29.370300 print_text (rostopic/rostopic)\n', '2025-01-19T02:09:29.370300 pub_text (rostopic/rostopic)\n', '2025-01-19T02:09:29.370300 rosbag_record (rosbag/record)\n', '2025-01-19T02:09:29.370300 \x1b[1mROS_MASTER_URI=http://localhost:11311\n', '2025-01-19T02:09:29.370300 \x1b[1mprocess[pub_text-22]: started with pid [2578397]\n', '2025-01-19T02:09:29.370300 \x1b[1mprocess[print_text-23]: started with pid [2578398]\n', '2025-01-19T02:09:29.370300 \x1b[1mprocess[rosbag_record-24]: started with pid [2578399]\n', '2025-01-19T02:09:29.370300 started bagging!\n']}}, 'dropped': 0}} +2025-01-18 21:09:44,930 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:09:44,930 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:09:45,077 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:45,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:46,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:47,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:48,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:49,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:50,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:50,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:51,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:52,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:53,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:54,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:55,117 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:55,175 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:09:55,176 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 with policy now +2025-01-18 21:09:55,217 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:09:55,217 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:09:55,218 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:09:55,218 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:09:55,471 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/ob9870et-media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 +2025-01-18 21:09:55,624 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:55,624 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:09:55,624 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 +2025-01-18 21:09:55,625 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:09:55,919 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/kqy5bfoe-media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 +2025-01-18 21:09:56,116 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:56,258 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:09:57,111 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:57,625 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:58,112 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:59,116 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:09:59,626 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:09:59,949 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:09:59,949 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:10:00,117 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:01,117 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:01,794 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:02,109 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:03,114 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:03,627 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:04,153 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:04,429 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:10:04,430 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:10:04,430 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:10:04,627 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:10:04,628 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:10:05,115 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:05,628 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:06,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:07,138 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:07,317 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:07,628 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:08,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:09,143 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:09,629 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:10,125 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:11,132 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:12,133 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:12,970 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:13,144 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:14,127 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:14,563 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:10:14,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:10:14,929 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:10:15,127 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:15,632 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:16,128 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:17,128 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:17,633 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:18,105 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:18,128 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:19,128 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:19,633 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:20,128 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:21,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:22,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:23,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:23,956 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:24,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:25,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:26,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:27,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:27,636 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:28,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:29,129 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:29,637 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:29,929 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:10:29,929 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:10:30,045 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:30,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:31,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:32,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:33,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:33,639 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:34,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:35,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:35,639 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:35,956 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:36,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:37,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:38,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:39,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:40,131 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:41,140 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:41,497 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:41,642 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:41,827 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:10:41,827 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 with policy now +2025-01-18 21:10:41,837 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:10:41,838 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:10:41,838 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:10:41,838 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:10:42,116 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/cwk3e2tk-media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 +2025-01-18 21:10:42,144 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:42,642 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:10:42,642 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 +2025-01-18 21:10:42,642 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:10:42,958 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/2hglpkjk-media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 +2025-01-18 21:10:43,150 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:43,642 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:44,167 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:44,563 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:10:44,982 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:10:44,982 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:10:45,154 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:46,165 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:46,553 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:47,211 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:47,644 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:48,173 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:49,178 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:49,487 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:10:49,487 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:10:49,492 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:10:49,645 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:49,645 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:10:49,645 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:10:49,903 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/z2qrvsct-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:10:50,167 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:51,171 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:51,646 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:51,982 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:52,189 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:53,173 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:53,646 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:54,195 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:55,180 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:56,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:57,196 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:57,974 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:10:58,199 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:59,192 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:10:59,649 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:10:59,935 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:10:59,935 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:11:00,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:01,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:01,650 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:02,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:03,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:03,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:04,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:05,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:05,652 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:06,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:07,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:07,652 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:08,194 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:08,245 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:09,195 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:10,195 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:11,195 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:12,195 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:13,195 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:13,961 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:14,196 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:14,564 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:11:14,935 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:11:14,936 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:11:15,196 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:16,196 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:17,196 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:18,196 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:19,095 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:19,196 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:20,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:21,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:21,657 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:22,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:23,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:24,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:24,458 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:25,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:26,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:27,221 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:27,660 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:27,811 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:11:27,811 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 with policy now +2025-01-18 21:11:27,814 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:11:27,814 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:11:27,814 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:11:27,814 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:11:28,082 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/0391rdg1-media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 +2025-01-18 21:11:28,209 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:28,660 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:11:28,661 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 +2025-01-18 21:11:28,661 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:11:28,977 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/qrkurpbu-media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 +2025-01-18 21:11:29,214 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:29,661 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:29,846 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:29,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:11:29,959 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:11:30,217 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:31,214 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:32,224 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:33,230 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:33,662 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:34,260 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:34,962 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:35,225 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:35,663 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:36,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:37,221 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:37,544 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:11:37,544 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:11:37,545 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:11:37,664 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:37,664 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:11:37,664 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:11:38,261 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:39,242 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:39,665 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:39,967 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:40,244 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:41,233 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:41,665 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:42,239 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:43,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:43,666 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:11:44,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:44,564 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:11:44,940 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:11:44,940 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:11:45,063 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:45,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:46,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:47,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:48,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:49,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:50,234 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:50,964 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:51,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:52,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:53,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:54,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:55,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:55,965 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:11:56,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:57,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:58,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:59,236 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:11:59,940 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:11:59,941 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:12:00,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:01,084 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:01,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:02,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:03,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:04,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:05,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:06,238 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:06,965 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:07,238 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:08,238 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:09,238 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:10,246 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:10,269 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:12:10,269 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 with policy now +2025-01-18 21:12:10,313 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:12:10,313 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:12:10,314 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:12:10,314 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:12:10,535 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/2n9jc6ae-media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 +2025-01-18 21:12:10,677 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:12:10,677 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 +2025-01-18 21:12:10,678 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:12:10,979 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/i301d7bv-media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 +2025-01-18 21:12:11,257 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:11,678 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:12,277 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:12,339 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:13,291 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:14,264 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:14,565 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:12:14,953 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:12:14,954 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:12:15,267 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:15,679 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:16,271 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:17,265 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:17,680 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:18,097 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:18,270 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:19,274 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:20,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:21,302 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:22,305 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:22,600 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:12:22,600 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:12:22,601 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:12:22,682 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:12:22,682 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:12:22,953 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/szyye1o9-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:12:23,284 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:23,566 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:23,682 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:24,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:25,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:25,683 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:26,299 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:27,299 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:27,684 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:28,314 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:28,705 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:29,296 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:29,971 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:12:29,972 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:12:30,308 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:31,309 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:32,299 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:33,297 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:34,066 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:34,300 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:35,293 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:35,687 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:36,294 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:37,294 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:37,688 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:38,294 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:39,130 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:39,294 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:40,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:41,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:42,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:43,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:44,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:44,517 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:44,565 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:12:44,945 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:12:44,945 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:12:45,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:45,691 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:46,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:47,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:48,295 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:49,296 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:49,988 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:50,296 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:51,296 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:52,296 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:53,296 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:54,296 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:54,988 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:12:55,298 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:56,307 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:56,448 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:12:56,448 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 with policy now +2025-01-18 21:12:56,451 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:12:56,451 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:12:56,451 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:12:56,452 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:12:56,696 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:12:56,696 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 +2025-01-18 21:12:56,696 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:12:56,718 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/nyybbuco-media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 +2025-01-18 21:12:57,015 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/ykakra7a-media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 +2025-01-18 21:12:57,305 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:57,696 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:12:58,346 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:59,359 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:12:59,966 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:12:59,967 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:13:00,083 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:00,335 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:01,356 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:01,698 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:02,361 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:03,381 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:04,368 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:04,652 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:13:04,652 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:13:04,653 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:13:04,699 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:13:04,699 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:13:05,395 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:05,442 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:05,699 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:06,382 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:07,390 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:07,700 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:08,400 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:09,407 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:09,701 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:10,425 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:11,001 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:11,406 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:12,428 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:13,433 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:14,429 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:14,566 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:13:14,988 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:13:14,988 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:13:15,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:16,105 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:16,420 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:17,461 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:17,704 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:18,420 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:19,420 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:19,705 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:20,420 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:21,420 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:21,995 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:22,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:23,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:24,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:25,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:26,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:26,995 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:27,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:28,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:29,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:29,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:13:29,960 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:13:30,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:31,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:32,094 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:32,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:33,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:34,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:35,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:36,422 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:37,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:37,995 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:38,425 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:39,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:40,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:41,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:42,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:42,995 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:43,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:44,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:44,566 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:13:44,974 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:13:44,975 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:13:45,419 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:13:45,419 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 with policy now +2025-01-18 21:13:45,439 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:45,444 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:13:45,444 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:13:45,444 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:13:45,445 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:13:45,702 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/qc4ilx14-media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 +2025-01-18 21:13:45,715 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:45,715 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:13:45,715 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 +2025-01-18 21:13:45,716 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:13:46,020 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/0ea2c26n-media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 +2025-01-18 21:13:46,445 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:47,438 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:47,716 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:48,467 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:48,507 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:49,499 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:49,717 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:50,457 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:51,455 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:52,439 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:53,472 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:53,718 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:13:53,909 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:54,458 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:55,468 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:56,457 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:57,470 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:58,475 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:59,017 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:13:59,477 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:13:59,998 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:13:59,998 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:14:00,477 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:01,482 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:02,515 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:03,505 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:03,722 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:14:03,722 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:14:03,723 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:14:03,723 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:14:03,724 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:14:04,524 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:04,724 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:14:04,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:05,492 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:05,724 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:06,515 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:07,497 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:08,523 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:09,548 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:09,726 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:09,837 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:10,525 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:11,551 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:12,522 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:13,540 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:14,545 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:14,567 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:14:14,971 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:14:14,972 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:14:15,126 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:15,561 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:16,571 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:17,538 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:17,729 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:18,538 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:19,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:19,730 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:20,538 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:21,060 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:21,538 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:21,731 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:22,538 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:23,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:24,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:25,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:26,061 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:26,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:27,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:28,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:29,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:29,963 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:14:29,963 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:14:30,539 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:31,098 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:31,540 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:32,540 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:33,540 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:34,540 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:35,540 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:36,540 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:37,061 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:37,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:38,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:39,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:40,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:41,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:42,061 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:42,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:43,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:44,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:44,567 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:14:44,963 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:14:44,963 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:14:45,577 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:45,741 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:46,154 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:14:46,154 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 with policy now +2025-01-18 21:14:46,157 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:14:46,157 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:14:46,157 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:14:46,158 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:14:46,444 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/w5wwaed3-media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 +2025-01-18 21:14:46,555 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:46,741 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:14:46,741 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 +2025-01-18 21:14:46,742 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:14:47,056 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/wnenej0g-media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 +2025-01-18 21:14:47,206 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:47,588 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:47,742 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:48,615 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:49,587 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:50,593 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:51,609 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:51,743 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:52,600 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:52,965 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:53,602 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:53,744 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:14:54,608 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:55,630 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:56,607 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:57,624 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:58,081 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:14:58,609 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:59,643 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:14:59,969 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:14:59,969 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:15:00,621 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:01,633 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:02,636 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:03,096 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:03,652 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:04,670 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:05,672 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:06,649 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:07,657 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:08,658 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:09,073 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:09,672 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:10,665 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:11,675 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:12,666 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:13,665 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:13,752 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:15:13,752 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:15:13,808 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:15:13,809 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:15:13,809 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:15:14,388 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:14,436 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/yyx1h8uq-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:15:14,567 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:15:14,692 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:14,753 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:15:14,978 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:15:14,978 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:15:15,723 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:15,753 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:15:16,690 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:17,688 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:18,720 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:19,569 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:19,713 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:19,755 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:15:20,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:21,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:22,715 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:23,727 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:24,735 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:25,116 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:25,761 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:26,801 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:27,777 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:28,728 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:29,728 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:29,759 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:15:29,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:15:29,976 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:15:30,728 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:31,102 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:31,728 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:31,760 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:15:32,728 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:33,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:34,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:35,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:36,102 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:36,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:37,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:38,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:39,732 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:40,729 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:41,103 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:41,730 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:42,730 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:43,730 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:44,568 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:15:44,730 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:44,976 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:15:44,976 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:15:45,730 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:46,118 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:46,730 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:47,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:48,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:49,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:50,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:51,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:52,103 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:52,731 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:53,732 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:54,733 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:55,733 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:56,752 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:57,197 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:15:57,363 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:15:57,363 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 with policy now +2025-01-18 21:15:57,377 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:15:57,378 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:15:57,378 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:15:57,378 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:15:57,643 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/p9egipbp-media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 +2025-01-18 21:15:57,739 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:57,770 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:15:57,770 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:15:57,771 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 +2025-01-18 21:15:57,771 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:15:58,066 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/pnz7oe18-media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 +2025-01-18 21:15:58,762 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:59,752 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:15:59,771 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:15:59,991 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:15:59,992 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:16:00,755 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:01,772 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:01,778 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:02,757 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:03,009 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:03,769 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:04,779 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:05,774 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:05,819 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:06,808 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:07,774 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:07,775 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:16:07,785 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:07,880 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:16:07,880 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:16:07,881 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:16:08,357 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:08,775 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:16:08,835 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:09,775 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:09,783 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:10,820 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:11,783 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:12,790 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:13,737 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:13,779 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:13,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:14,568 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:16:14,822 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:14,981 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:16:14,982 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:16:15,810 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:16,841 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:17,837 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:18,824 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:18,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:19,781 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:19,810 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:20,810 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:21,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:22,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:23,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:24,122 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:24,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:25,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:26,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:27,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:28,811 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:29,122 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:29,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:29,979 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:16:29,979 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:16:30,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:31,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:32,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:33,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:34,123 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:34,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:35,812 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:36,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:37,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:38,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:39,123 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:39,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:40,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:41,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:42,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:43,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:44,569 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:16:44,570 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:44,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:44,979 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:16:44,979 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:16:45,813 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:46,831 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:47,097 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:16:47,097 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 with policy now +2025-01-18 21:16:47,117 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:16:47,117 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:16:47,117 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:16:47,118 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:16:47,368 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/psj3ohm0-media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 +2025-01-18 21:16:47,797 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:47,797 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:16:47,797 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 +2025-01-18 21:16:47,797 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:16:47,864 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:48,117 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/y5yntkla-media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 +2025-01-18 21:16:48,854 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:49,849 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:50,127 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:50,905 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:51,798 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:51,851 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:52,854 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:53,848 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:54,847 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:55,139 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:16:55,860 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:56,800 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:16:56,884 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:57,877 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:58,903 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:16:59,903 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:00,029 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:17:00,029 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:17:00,902 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:01,138 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:01,941 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:02,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:03,803 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:17:03,948 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:03,948 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:17:03,948 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:17:03,948 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:17:04,803 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:04,804 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:17:04,907 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:05,935 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:06,804 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:06,936 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:07,138 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:07,923 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:08,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:09,935 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:10,806 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:10,948 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:11,933 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:12,167 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:12,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:13,969 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:14,569 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:17:14,998 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:17:14,998 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:17:15,045 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:16,010 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:16,952 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:17,541 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:17,963 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:18,809 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:18,956 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:19,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:20,810 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:20,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:21,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:22,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:23,103 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:23,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:24,812 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:24,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:25,957 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:26,813 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:26,958 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:27,958 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:28,545 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:28,814 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:28,958 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:29,958 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:29,985 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:17:29,985 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:17:30,958 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:31,958 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:32,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:33,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:34,131 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:34,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:35,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:36,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:37,959 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:38,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:39,131 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:39,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:40,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:41,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:42,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:43,960 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:44,131 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:44,570 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:17:44,993 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:45,004 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:17:45,004 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:17:45,975 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:45,998 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:17:45,998 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 with policy now +2025-01-18 21:17:46,041 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:17:46,042 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:17:46,042 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:17:46,046 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:17:46,296 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/05pnzh1z-media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 +2025-01-18 21:17:46,822 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:46,822 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:17:46,822 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 +2025-01-18 21:17:46,822 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:17:46,973 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:47,145 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/xnrnd6np-media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 +2025-01-18 21:17:47,822 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:47,997 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:48,823 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:49,004 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:49,734 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:49,995 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:50,984 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:52,026 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:52,824 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:53,002 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:54,027 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:54,825 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:17:55,043 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:55,172 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:17:56,071 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:57,033 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:58,046 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:59,046 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:17:59,994 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:17:59,994 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:18:00,055 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:00,421 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:00,828 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:18:01,021 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:18:01,021 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:18:01,021 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:18:01,022 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:01,089 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/tncyld2n-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:18:01,828 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:18:02,024 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:02,829 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:03,067 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:04,030 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:04,829 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:05,044 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:06,078 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:06,184 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:06,830 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:07,054 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:08,059 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:09,077 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:10,054 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:11,081 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:12,076 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:12,182 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:13,084 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:14,073 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:14,570 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:18:15,067 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:18:15,067 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:18:15,109 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:16,147 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:17,113 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:18,107 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:18,179 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:19,089 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:20,089 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:20,836 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:21,089 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:22,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:22,838 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:23,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:24,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:24,152 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:25,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:26,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:27,090 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:28,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:29,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:29,152 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:29,988 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:18:29,988 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:18:30,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:31,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:32,091 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:33,094 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:34,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:34,181 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:35,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:36,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:37,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:38,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:39,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:40,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:40,151 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:41,093 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:42,094 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:43,094 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:44,094 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:44,571 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:18:44,989 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:18:44,989 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:18:45,094 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:45,152 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:46,094 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:47,114 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:47,603 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:18:47,603 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 with policy now +2025-01-18 21:18:47,609 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:18:47,610 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:18:47,610 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:18:47,610 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:18:47,850 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:18:47,850 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 +2025-01-18 21:18:47,850 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:18:47,933 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/p7cj1ipe-media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 +2025-01-18 21:18:48,120 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:48,231 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/yi5afswe-media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 +2025-01-18 21:18:48,850 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:49,127 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:50,152 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:50,155 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:50,851 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:51,177 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:52,149 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:52,852 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:53,184 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:54,171 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:55,157 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:18:55,158 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:56,188 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:56,854 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:18:57,186 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:58,199 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:18:59,226 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:00,012 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:19:00,012 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:19:00,181 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:00,216 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:01,184 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:02,186 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:03,235 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:04,198 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:05,208 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:06,180 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:06,237 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:06,858 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:19:07,099 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:19:07,099 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:19:07,100 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:19:07,208 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:07,858 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:19:07,859 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:19:08,215 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:08,859 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:09,243 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:10,229 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:10,860 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:11,223 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:11,428 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:12,232 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:12,861 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:13,235 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:14,267 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:14,577 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:19:15,059 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:19:15,059 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:19:15,283 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:16,268 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:17,172 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:17,259 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:18,269 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:19,260 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:20,287 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:21,262 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:22,191 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:22,330 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:23,308 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:24,274 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:24,866 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:25,274 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:26,274 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:26,867 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:27,277 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:28,180 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:28,275 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:29,275 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:29,995 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:19:29,995 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:19:30,275 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:31,276 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:32,276 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:33,181 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:33,276 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:34,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:35,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:36,279 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:37,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:38,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:39,181 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:39,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:40,281 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:41,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:42,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:43,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:44,181 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:44,280 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:44,571 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:19:44,999 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:19:44,999 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:19:45,281 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:46,282 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:47,285 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:48,281 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:49,181 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:49,289 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:50,285 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:51,286 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:52,326 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:53,090 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:19:53,090 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 with policy now +2025-01-18 21:19:53,101 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:19:53,101 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:19:53,101 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:19:53,103 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:19:53,328 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:53,371 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/cy0ojnxt-media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 +2025-01-18 21:19:53,879 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:53,879 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:19:53,879 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 +2025-01-18 21:19:53,879 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:19:54,239 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:19:54,245 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/s7eojymz-media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 +2025-01-18 21:19:54,343 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:54,879 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:55,331 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:56,344 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:57,326 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:58,343 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:58,881 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:19:59,307 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:19:59,913 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:00,036 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:20:00,036 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:20:00,306 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:01,330 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:02,342 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:02,883 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:03,381 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:04,369 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:05,198 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:05,356 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:06,396 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:07,436 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:07,885 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:20:08,171 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:20:08,171 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:20:08,172 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:20:08,381 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:08,886 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:08,886 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:20:08,886 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:20:09,382 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:10,247 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:10,395 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:10,886 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:11,386 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:12,389 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:13,415 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:14,410 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:14,572 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:20:14,888 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:15,020 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:20:15,020 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:20:15,251 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:15,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:16,404 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:17,443 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:18,416 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:19,416 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:20,449 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:21,215 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:21,448 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:22,442 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:23,447 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:24,445 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:25,434 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:26,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:26,893 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:26,915 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:27,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:28,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:28,894 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:29,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:30,011 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:20:30,012 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:20:30,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:31,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:32,212 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:32,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:33,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:34,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:35,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:36,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:37,424 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:38,213 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:38,425 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:39,425 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:40,425 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:41,425 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:42,426 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:42,901 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:43,293 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:43,426 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:44,426 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:44,572 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:20:45,011 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:20:45,012 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:20:45,426 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:46,426 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:47,427 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:48,427 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:49,213 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:49,426 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:50,427 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:51,427 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:52,427 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:53,486 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:53,965 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:20:53,966 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 with policy now +2025-01-18 21:20:53,987 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:20:53,988 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:20:53,988 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:20:53,988 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:20:54,277 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/g1t2eo87-media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 +2025-01-18 21:20:54,438 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:54,906 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:54,907 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:20:54,907 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 +2025-01-18 21:20:54,907 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval +2025-01-18 21:20:55,026 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:20:55,220 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/pvmpldhg-media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 +2025-01-18 21:20:55,478 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:55,907 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:56,467 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:56,907 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:57,496 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:58,489 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:20:58,908 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:20:59,493 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:00,020 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:21:00,020 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:21:00,115 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:00,517 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:00,909 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:21:01,518 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:02,512 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:02,910 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:21:03,531 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:04,548 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:05,285 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:05,544 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:06,554 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:07,553 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:08,585 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:09,243 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:21:09,244 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:09,244 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:09,613 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:09,913 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:21:09,913 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:21:10,609 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:10,908 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 21:21:10,908 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 21:21:10,914 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:21:11,137 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:21:11,342 INFO wandb-upload_2:2577265 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp2x7lu8zi +2025-01-18 21:21:11,593 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:11,744 INFO SenderThread:2577265 [sender.py:send_request_log_artifact():1525] logged artifact run-2t6elatx-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MjA0NTYxNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzExNTAzNA==', 'latestArtifact': None}} +2025-01-18 21:21:11,744 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:21:11,744 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json with policy now +2025-01-18 21:21:11,745 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:11,745 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:21:11,745 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:11,746 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:11,914 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:21:11,914 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json +2025-01-18 21:21:11,914 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media +2025-01-18 21:21:11,915 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table +2025-01-18 21:21:11,915 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval +2025-01-18 21:21:11,940 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/hzqzkb5b-media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json +2025-01-18 21:21:12,206 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/e7jjem0a-media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json +2025-01-18 21:21:12,593 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:13,594 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:14,573 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:21:14,594 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:15,016 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:21:15,016 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:21:15,595 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:16,596 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:17,233 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:17,595 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:18,596 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:19,596 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:20,596 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:21,596 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:22,597 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:22,893 DEBUG SenderThread:2577265 [sender.py:send():379] send: telemetry +2025-01-18 21:21:22,894 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:22,919 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:21:22,941 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,941 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,941 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,942 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,942 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,942 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,943 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,943 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,943 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,944 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,944 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,944 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,945 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,945 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,945 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,946 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,946 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,946 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,946 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,947 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:22,947 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:22,947 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:23,034 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 21:21:23,034 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 21:21:23,044 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:21:23,188 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/ahcq43fd-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:21:23,382 INFO wandb-upload_2:2577265 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpcblxnubg +2025-01-18 21:21:23,597 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:23,800 INFO SenderThread:2577265 [sender.py:send_request_log_artifact():1525] logged artifact run-2t6elatx-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MjA0NTczOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzExNTA1Nw==', 'latestArtifact': None}} +2025-01-18 21:21:23,800 DEBUG SenderThread:2577265 [sender.py:send():379] send: files +2025-01-18 21:21:23,800 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json with policy now +2025-01-18 21:21:23,800 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:21:23,800 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:23,801 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:23,920 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:21:23,921 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:21:23,921 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json +2025-01-18 21:21:23,921 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval +2025-01-18 21:21:23,999 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/mr0gm6ct-media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json +2025-01-18 21:21:24,280 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/d6rv1i6i-media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json +2025-01-18 21:21:24,598 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:24,921 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:21:25,597 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:26,597 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:27,598 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:28,233 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:28,599 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:29,597 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:29,689 DEBUG SenderThread:2577265 [sender.py:send():379] send: exit +2025-01-18 21:21:29,689 INFO SenderThread:2577265 [sender.py:send_exit():586] handling exit code: 0 +2025-01-18 21:21:29,690 INFO SenderThread:2577265 [sender.py:send_exit():588] handling runtime: 1065 +2025-01-18 21:21:29,691 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:29,691 INFO SenderThread:2577265 [sender.py:send_exit():594] send defer +2025-01-18 21:21:29,691 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:29,691 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-18 21:21:29,691 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:29,691 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-18 21:21:29,691 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 1 +2025-01-18 21:21:29,691 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:29,691 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-18 21:21:29,692 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:29,692 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-18 21:21:29,692 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 2 +2025-01-18 21:21:29,692 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:29,692 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-18 21:21:29,692 INFO HandlerThread:2577265 [system_monitor.py:finish():203] Stopping system monitor +2025-01-18 21:21:29,692 DEBUG SystemMonitor:2577265 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-18 21:21:29,692 DEBUG SystemMonitor:2577265 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-18 21:21:29,693 INFO HandlerThread:2577265 [interfaces.py:finish():200] Joined cpu monitor +2025-01-18 21:21:29,693 INFO HandlerThread:2577265 [interfaces.py:finish():200] Joined disk monitor +2025-01-18 21:21:29,744 INFO HandlerThread:2577265 [interfaces.py:finish():200] Joined gpu monitor +2025-01-18 21:21:29,744 INFO HandlerThread:2577265 [interfaces.py:finish():200] Joined memory monitor +2025-01-18 21:21:29,744 INFO HandlerThread:2577265 [interfaces.py:finish():200] Joined network monitor +2025-01-18 21:21:29,744 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:29,744 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-18 21:21:29,744 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 3 +2025-01-18 21:21:29,745 DEBUG SenderThread:2577265 [sender.py:send():379] send: stats +2025-01-18 21:21:29,745 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:29,745 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-18 21:21:29,745 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:29,745 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-18 21:21:29,745 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 4 +2025-01-18 21:21:29,745 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:29,745 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-18 21:21:29,923 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:21:30,924 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:21:36,270 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:21:36,270 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:36,270 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: keepalive +2025-01-18 21:21:36,271 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:36,271 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-18 21:21:36,271 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 5 +2025-01-18 21:21:36,271 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:21:36,271 DEBUG SenderThread:2577265 [sender.py:send():379] send: history +2025-01-18 21:21:36,271 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:21:36,272 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:36,272 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-18 21:21:36,272 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:36,272 DEBUG SenderThread:2577265 [sender.py:send():379] send: summary +2025-01-18 21:21:36,276 INFO SenderThread:2577265 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:21:36,277 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:36,277 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-18 21:21:36,277 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 6 +2025-01-18 21:21:36,277 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:36,277 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-18 21:21:36,277 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:36,277 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-18 21:21:36,283 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:21:36,426 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 7 +2025-01-18 21:21:36,427 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:36,427 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-18 21:21:36,427 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:36,427 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-18 21:21:36,691 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:21:36,927 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/config.yaml +2025-01-18 21:21:36,927 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:21:39,909 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 8 +2025-01-18 21:21:39,909 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:21:39,910 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:39,910 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-18 21:21:39,910 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:39,910 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-18 21:21:39,910 INFO SenderThread:2577265 [job_builder.py:build():440] Attempting to build job artifact +2025-01-18 21:21:39,913 INFO SenderThread:2577265 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-18 21:21:39,915 INFO SenderThread:2577265 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-18 21:21:39,928 INFO Thread-12 :2577265 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:21:39,934 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 9 +2025-01-18 21:21:39,934 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:39,934 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-18 21:21:39,935 DEBUG SenderThread:2577265 [sender.py:send():379] send: artifact +2025-01-18 21:21:40,322 INFO wandb-upload_1:2577265 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpe1wv_oo5 +2025-01-18 21:21:40,322 INFO wandb-upload_2:2577265 [upload_job.py:push():85] Skipped uploading /tmp/tmpfj0s81cv/wandb-job.json +2025-01-18 21:21:40,506 INFO wandb-upload_3:2577265 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp8gzxv2_4 +2025-01-18 21:21:40,692 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:21:41,186 INFO SenderThread:2577265 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1MjA0NTk0MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1MjAyMTE2MQ==', 'versionIndex': 450}}} +2025-01-18 21:21:41,186 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:41,186 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-18 21:21:41,186 INFO SenderThread:2577265 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-18 21:21:41,929 INFO SenderThread:2577265 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files +2025-01-18 21:21:41,929 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/config.yaml config.yaml +2025-01-18 21:21:41,930 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json wandb-summary.json +2025-01-18 21:21:41,930 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:21:41,930 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/requirements.txt requirements.txt +2025-01-18 21:21:41,930 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-metadata.json wandb-metadata.json +2025-01-18 21:21:41,930 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 21:21:41,930 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log output.log +2025-01-18 21:21:41,932 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/diff.patch diff.patch +2025-01-18 21:21:41,933 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-18 21:21:41,936 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 media/videos/eval/17_eval_17_cb16d97a7d17e520d767.mp4 +2025-01-18 21:21:41,936 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 media/videos/eval/16_eval_16_dec0060310bb185ed9e7.mp4 +2025-01-18 21:21:41,936 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 media/videos/eval/13_eval_13_f717f47f7f63a8b77820.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 media/videos/eval/0_eval_0_373935c9a53d50a21516.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 media/videos/eval/2_eval_2_9019146f9223df893e27.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 media/videos/eval/1_eval_1_f136ddfa6a75f8c38319.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 media/videos/eval/15_eval_15_7379e8bc33b5c38d196c.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 media/videos/eval/9_eval_9_0cb54919d1152aace357.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 media/videos/eval/8_eval_8_4562dd864115606b6291.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 media/videos/eval/10_eval_10_194dcd6049c4233da56c.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 media/videos/eval/14_eval_14_6cb31531142c32910aad.mp4 +2025-01-18 21:21:41,938 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 media/videos/eval/12_eval_12_8a945bf8da73837c5024.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 media/videos/eval/7_eval_7_d9b2fef5e246e9ba0a8b.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 media/videos/eval/6_eval_6_a90b503bf7614a8b51c5.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 media/videos/eval/19_eval_19_0ae4f8c9bd51774c0b89.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 media/videos/eval/18_eval_18_c883e9e40544a093c095.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 media/videos/eval/11_eval_11_136a95e750557f39dd09.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 media/videos/eval/4_eval_4_ff051e1686bf761b709a.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 media/videos/eval/5_eval_5_1a0fa497056b618ea0c0.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 media/videos/eval/3_eval_3_a9352300f4a1f2d4fc59.mp4 +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json media/table/eval/total_success_rate_ci_21_984e345eb950dc10e24c.table.json +2025-01-18 21:21:41,939 INFO SenderThread:2577265 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json media/table/eval/success_rate_ci_20_046a987f9352685cc2d7.table.json +2025-01-18 21:21:41,940 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 10 +2025-01-18 21:21:41,940 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:21:41,940 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:41,940 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-18 21:21:41,940 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:41,940 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-18 21:21:41,940 INFO SenderThread:2577265 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 21:21:42,131 INFO wandb-upload_0:2577265 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/config.yaml +2025-01-18 21:21:42,147 INFO wandb-upload_1:2577265 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/wandb-summary.json +2025-01-18 21:21:42,163 INFO wandb-upload_4:2577265 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/output.log +2025-01-18 21:21:42,171 INFO wandb-upload_2:2577265 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/files/requirements.txt +2025-01-18 21:21:42,223 INFO wandb-upload_3:2577265 [upload_job.py:push():130] Uploaded file /tmp/tmplz8rujtfwandb/b03309oi-events.out.tfevents.1737252224.leonmkim-ROG-Strix-G15CS-G15CS.2577081.0 +2025-01-18 21:21:42,423 INFO Thread-11 (_thread_body):2577265 [sender.py:transition_state():614] send defer: 11 +2025-01-18 21:21:42,423 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:42,423 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-18 21:21:42,424 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:42,424 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-18 21:21:42,424 INFO SenderThread:2577265 [file_pusher.py:join():175] waiting for file pusher +2025-01-18 21:21:42,427 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 12 +2025-01-18 21:21:42,427 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:42,427 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-18 21:21:42,427 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:42,427 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-18 21:21:42,427 INFO SenderThread:2577265 [file_stream.py:finish():601] file stream finish called +2025-01-18 21:21:42,661 INFO SenderThread:2577265 [file_stream.py:finish():605] file stream finish is done +2025-01-18 21:21:42,661 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 13 +2025-01-18 21:21:42,661 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:42,661 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-18 21:21:42,661 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:42,661 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-18 21:21:42,661 INFO SenderThread:2577265 [sender.py:transition_state():614] send defer: 14 +2025-01-18 21:21:42,662 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:21:42,662 INFO HandlerThread:2577265 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-18 21:21:42,662 DEBUG SenderThread:2577265 [sender.py:send():379] send: final +2025-01-18 21:21:42,662 DEBUG SenderThread:2577265 [sender.py:send():379] send: footer +2025-01-18 21:21:42,662 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: defer +2025-01-18 21:21:42,662 INFO SenderThread:2577265 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-18 21:21:42,662 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:21:42,663 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:21:42,663 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: server_info +2025-01-18 21:21:42,663 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: server_info +2025-01-18 21:21:42,664 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:21:42,665 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: get_summary +2025-01-18 21:21:42,665 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-18 21:21:42,666 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:21:42,719 DEBUG SenderThread:2577265 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:21:42,719 INFO MainThread:2577265 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-18 21:21:42,720 INFO MainThread:2577265 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-18 21:21:42,720 INFO MainThread:2577265 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-18 21:21:42,721 DEBUG HandlerThread:2577265 [handler.py:handle_request():158] handle_request: shutdown +2025-01-18 21:21:42,721 INFO HandlerThread:2577265 [handler.py:finish():882] shutting down handler +2025-01-18 21:21:43,665 INFO WriterThread:2577265 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/run-2t6elatx.wandb +2025-01-18 21:21:43,719 INFO SenderThread:2577265 [sender.py:finish():1615] shutting down sender +2025-01-18 21:21:43,719 INFO SenderThread:2577265 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 21:21:43,720 INFO SenderThread:2577265 [file_pusher.py:join():175] waiting for file pusher diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/logs/debug.log b/2t6elatx/wandb/run-20250118_210344-2t6elatx/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..34d3c18d2893ce810a4a9160a32d1df30b8ee614 --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-18 21:03:44,031 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-18 21:03:44,031 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Configure stats pid to 2577081 +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/settings +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/logs/debug.log +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/wandb/run-20250118_210344-2t6elatx/logs/debug-internal.log +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_init.py:init():569] calling init triggers +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 2, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_2', 'wandb_notes': '3465_2_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 2, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx'} +2025-01-18 21:03:44,032 INFO MainThread:2577081 [wandb_init.py:init():619] starting backend +2025-01-18 21:03:44,033 INFO MainThread:2577081 [wandb_init.py:init():623] setting up manager +2025-01-18 21:03:44,036 INFO MainThread:2577081 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-18 21:03:44,037 INFO MainThread:2577081 [wandb_init.py:init():631] backend started and connected +2025-01-18 21:03:44,051 INFO MainThread:2577081 [wandb_init.py:init():720] updated telemetry +2025-01-18 21:03:44,059 INFO MainThread:2577081 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-18 21:03:44,500 INFO MainThread:2577081 [wandb_run.py:_on_init():2435] communicating current version +2025-01-18 21:03:44,541 INFO MainThread:2577081 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-18 21:03:44,541 INFO MainThread:2577081 [wandb_init.py:init():804] starting run threads in backend +2025-01-18 21:03:44,904 INFO MainThread:2577081 [wandb_run.py:_console_start():2413] atexit reg +2025-01-18 21:03:44,904 INFO MainThread:2577081 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-18 21:03:44,904 INFO MainThread:2577081 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-18 21:03:44,904 INFO MainThread:2577081 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-18 21:03:44,905 INFO MainThread:2577081 [wandb_init.py:init():847] run started, returning control to user process +2025-01-18 21:03:44,906 INFO MainThread:2577081 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/2t6elatx/tb, True +2025-01-18 21:03:48,880 INFO MainThread:2577081 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-18 21:21:43,722 WARNING MsgRouterThr:2577081 [router.py:message_loop():77] message_loop has been closed diff --git a/2t6elatx/wandb/run-20250118_210344-2t6elatx/run-2t6elatx.wandb b/2t6elatx/wandb/run-20250118_210344-2t6elatx/run-2t6elatx.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c5fc9f5c1ed194fa902e30b36536230c24c2339f --- /dev/null +++ b/2t6elatx/wandb/run-20250118_210344-2t6elatx/run-2t6elatx.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:282ddf51b1b1be8a3c1df0b42e48172ddafcb2ac03b06e84028a47176a3762ba +size 229159 diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e2f570dd2dc02c3daa465b3812643325c1eecc5 --- /dev/null +++ b/config.yaml @@ -0,0 +1,573 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 2 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '3465_2' +wandb_notes: 3465_2_restarted_1 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: false +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 7 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +restore_missing_checkpoint: null +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ${test.root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${test.name_of_expert_demo} + expert_dataset: ${test.expert_dataset_dirpath}/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ${test.expert_dataset_dirpath}/${test.semantic_demo_grouping_name} + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: ${test.local_snapshot_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name} + resume_wandb_run: false +feature_type: 180x240_1_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/lbb9nq1r/.hydra/config.yaml b/lbb9nq1r/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e205199f663cb5a4be0ac1c8b8e854ceb56738d5 --- /dev/null +++ b/lbb9nq1r/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3465_2' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/lbb9nq1r/.hydra/hydra.yaml b/lbb9nq1r/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e06f855c34f5b6ce8ef296e94882867eef9ec28 --- /dev/null +++ b/lbb9nq1r/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_2/lbb9nq1r + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/lbb9nq1r/.hydra/overrides.yaml b/lbb9nq1r/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/lbb9nq1r/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/lbb9nq1r/eval_policy.log b/lbb9nq1r/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..cc27b5ba0a2457cd9757906e765139fd79e0e60f --- /dev/null +++ b/lbb9nq1r/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-18 21:03:34,238][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-18 21:03:34,242][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-18 21:03:38,404][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/snapshot_10499.pt b/snapshot_10499.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce832150c445dd57ca6e4ac19692ecd8df64a19b --- /dev/null +++ b/snapshot_10499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a32993507118dcf0d549b4c7cc066cddfdd835e494495e4dc36252ed8b7a5ce5 +size 909989782 diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..752d2b41d24d143f1ca542144566ee1e48fc9071 --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed60e836446dee7a32b9b6c3be3027f45113b4e7c7a41545d65201127c893d7b +size 909989782 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2aa60502def3047e43275590519a950e9cacbf5 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1aef4dc6bf2238a99909573fcefaaa46dcb57937a9a38bce4c410eda94d44229 +size 909989782 diff --git a/snapshot_12749.pt b/snapshot_12749.pt new file mode 100644 index 0000000000000000000000000000000000000000..b81235219b95886bef956d3fc79287ffae8342b1 --- /dev/null +++ b/snapshot_12749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7780d5653821b6da9496616823ec653f19d405c40d71c34f8d4fc7dbeb76645 +size 909989782 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5628f88885c8b1be988770e31e3f747bea96756 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40418114d0d87fa838117ec332fbc081d8a8b8c827d23fe5b38f81428650a445 +size 909989782 diff --git a/snapshot_14749.pt b/snapshot_14749.pt new file mode 100644 index 0000000000000000000000000000000000000000..7615293cc8935dedc64ce52a237ebcf252386643 --- /dev/null +++ b/snapshot_14749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:699dea36debbf00116c828431fcb0b5ab93c8197130f8e85ce43603fe7d4a62d +size 909989782 diff --git a/snapshot_14999.pt b/snapshot_14999.pt new file mode 100644 index 0000000000000000000000000000000000000000..294ee4c9e0963638011128103c39e0ca619c632a --- /dev/null +++ b/snapshot_14999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0322779bdf5b0e242d46d855010eb707dc4a2a129f56a1bfba1cccb4b512c214 +size 909989782 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..72af29a1edb9469b2961591e9002644e3a7ea8a0 --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90a76b217edf8298c968828ccecd9edda088f83aaae999d9a267acad8b145b28 +size 909989782 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee274debac8206df577d623ad36889edf41555a6 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d33179eb858de3e7e011ddee933f764e0fe1d2f8d4f0792d16521f04a0c42a8 +size 909989336 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..bed662fdee44aa7032d4d8bdcbf31d749373b283 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d617d86a66f024cd8d8edbacd946b10b57dfcd2eba712d806e96f10a86181d5a +size 909989336 diff --git a/snapshot_7249.pt b/snapshot_7249.pt new file mode 100644 index 0000000000000000000000000000000000000000..997571ee15faf3a0b36bf5b5e38ead6590571292 --- /dev/null +++ b/snapshot_7249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65869b47ce4a9d914441865427d69c051885001b40ee6761e38fe7065f7e9b80 +size 909989336 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f752159ad9163f21cde37ef3cd8c7db88985d42 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67b9ca942d528c9c200cadfb07faee04ee976dd3bdb0c06f53299b3d8feebee4 +size 909989336 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..541525c3e9359cc058b4e108c82877e9e9b703d7 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b056487e4a626ae0b945ff9f80eccc7f56ba0d19985ea2deeef754d6a4e8075 +size 909989782 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..72af29a1edb9469b2961591e9002644e3a7ea8a0 --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90a76b217edf8298c968828ccecd9edda088f83aaae999d9a267acad8b145b28 +size 909989782