diff --git a/.gitattributes b/.gitattributes index a86a6110976194a7fc5cf2f700b4473c0012b8a0..592237fa9d042f0e058d4a100de9b4c68d82e9a6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -150,3 +150,64 @@ odxnffav/episode_rosbags/episode_5_2025-01-17-14-33-59.bag filter=lfs diff=lfs m odxnffav/episode_rosbags/episode_11_2025-01-17-14-38-18.bag filter=lfs diff=lfs merge=lfs -text odxnffav/episode_rosbags/episode_6_2025-01-17-14-34-44.bag filter=lfs diff=lfs merge=lfs -text odxnffav/episode_rosbags/episode_18_2025-01-17-14-43-55.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_8_2025-01-17-14-36-13.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_3_2025-01-17-14-32-34.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_17_2025-01-17-14-43-02.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_10_2025-01-17-14-37-39.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/run-odxnffav.wandb filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_1_2025-01-17-14-31-10.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_14_2025-01-15-21-37-17.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_12_2025-01-15-21-35-47.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/13_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/18_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/15_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_1_2025-01-15-21-27-32.bag filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_18_2025-01-15-21-40-48.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/14_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/16_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/eval_video/17_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_19_2025-01-17-14-44-48.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_15_2025-01-17-14-41-24.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_16_2025-01-17-14-42-10.bag filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_2_2025-01-15-21-28-26.bag filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_11_2025-01-15-21-34-54.bag filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_15_2025-01-15-21-38-03.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_4_2025-01-17-14-33-16.bag filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_8_2025-01-15-21-32-40.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_2_2025-01-17-14-31-51.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_13_2025-01-17-14-39-56.bag filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_0_2025-01-15-21-26-31.bag filter=lfs diff=lfs merge=lfs -text diff --git a/51cdtujf/.hydra/config.yaml b/51cdtujf/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c6218867bdef1ad3ae0c2d6cc61cdafef42eee2 --- /dev/null +++ b/51cdtujf/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3456_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/51cdtujf/.hydra/hydra.yaml b/51cdtujf/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..992cf19b2b770ca2666dca1d8009721d46407f8b --- /dev/null +++ b/51cdtujf/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/51cdtujf + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/51cdtujf/.hydra/overrides.yaml b/51cdtujf/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/51cdtujf/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/51cdtujf/eval_policy.log b/51cdtujf/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..b2149b75799daa9fe7445a40a9eef8ecec632b80 --- /dev/null +++ b/51cdtujf/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-17 14:29:20,547][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-17 14:29:20,550][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-17 14:29:24,035][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/odxnffav/.hydra/config.yaml b/odxnffav/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c6218867bdef1ad3ae0c2d6cc61cdafef42eee2 --- /dev/null +++ b/odxnffav/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3456_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/odxnffav/.hydra/hydra.yaml b/odxnffav/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d1a1f3adb674ba0cfcc3a3e1dbdef8a507faee5 --- /dev/null +++ b/odxnffav/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/odxnffav/.hydra/overrides.yaml b/odxnffav/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/odxnffav/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/odxnffav/episode_rosbags/aligned_depth_to_color_K.npy b/odxnffav/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/odxnffav/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/odxnffav/episode_rosbags/color_K.npy b/odxnffav/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/odxnffav/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/odxnffav/episode_rosbags/episode_10_2025-01-17-14-37-39.bag b/odxnffav/episode_rosbags/episode_10_2025-01-17-14-37-39.bag new file mode 100644 index 0000000000000000000000000000000000000000..fe6c59ad5904aa7cb8ec7ec53eb50f2748fc0ec1 --- /dev/null +++ b/odxnffav/episode_rosbags/episode_10_2025-01-17-14-37-39.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8b15951c1cc6085b1b5d968853432db2d974e14e1c28d6108ff1c537fd88613 +size 855723326 diff --git a/odxnffav/episode_rosbags/episode_13_2025-01-17-14-39-56.bag b/odxnffav/episode_rosbags/episode_13_2025-01-17-14-39-56.bag new file mode 100644 index 0000000000000000000000000000000000000000..1100fd4bad54b2cc9a183a1e2c4e7375da3b3293 --- /dev/null +++ b/odxnffav/episode_rosbags/episode_13_2025-01-17-14-39-56.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc4715813098a6d87368599d5e2fa2257f47456e472b77f103377f14f1c4c2a8 +size 1413630299 diff --git a/odxnffav/episode_rosbags/episode_15_2025-01-17-14-41-24.bag b/odxnffav/episode_rosbags/episode_15_2025-01-17-14-41-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..2b10983bff6363d7e57a2d31fc90bb0090f5a0b3 --- /dev/null +++ b/odxnffav/episode_rosbags/episode_15_2025-01-17-14-41-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47029e828ffc86ba5a0fdaf0e4a9396512c4ba8013f5a5376d82ffa94233e1f0 +size 1413224625 diff --git a/odxnffav/episode_rosbags/episode_16_2025-01-17-14-42-10.bag b/odxnffav/episode_rosbags/episode_16_2025-01-17-14-42-10.bag new file mode 100644 index 0000000000000000000000000000000000000000..1ab32cc752935ed248120f4b8c1515f3dfeb6860 --- /dev/null +++ b/odxnffav/episode_rosbags/episode_16_2025-01-17-14-42-10.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b98bb96621fffd69d9a83f9eaceb9a6b1a6e7cf824a66386c7608066c47d09dd +size 1409635546 diff --git a/odxnffav/episode_rosbags/episode_17_2025-01-17-14-43-02.bag b/odxnffav/episode_rosbags/episode_17_2025-01-17-14-43-02.bag new file mode 100644 index 0000000000000000000000000000000000000000..3112c7d849da096bcc52625ce222db1d56ed685c --- /dev/null +++ b/odxnffav/episode_rosbags/episode_17_2025-01-17-14-43-02.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ab8a6dc3391e0df4751bc595ee4deaa9c8b98db0ef2926e86bc3dd61a464831 +size 1412378324 diff --git a/odxnffav/episode_rosbags/episode_19_2025-01-17-14-44-48.bag b/odxnffav/episode_rosbags/episode_19_2025-01-17-14-44-48.bag new file mode 100644 index 0000000000000000000000000000000000000000..59f212d99be4d0e28e902e5338d9227270634035 --- /dev/null +++ b/odxnffav/episode_rosbags/episode_19_2025-01-17-14-44-48.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b88963efabf68df345725f6b2398e770e38dcf5f8d7fbbc154395531c5f6b907 +size 1415677252 diff --git a/odxnffav/episode_rosbags/episode_1_2025-01-17-14-31-10.bag b/odxnffav/episode_rosbags/episode_1_2025-01-17-14-31-10.bag new file mode 100644 index 0000000000000000000000000000000000000000..ba9c13d48aec76d761ab963e7cb7ec0a11694b72 --- /dev/null +++ b/odxnffav/episode_rosbags/episode_1_2025-01-17-14-31-10.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:019fce4537f1cf8741f0fcb0bf15aee28e20d20ffaf2bbaf7decb9239bcfd1ec +size 1413675345 diff --git a/odxnffav/episode_rosbags/episode_2_2025-01-17-14-31-51.bag b/odxnffav/episode_rosbags/episode_2_2025-01-17-14-31-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..fd684708f7c3ed49b4edda58a003fa7f901ad2c7 --- /dev/null +++ b/odxnffav/episode_rosbags/episode_2_2025-01-17-14-31-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fedc47324c78fba99d83049c094b80fc2775fc5965892249d8af0c6cc8df6f42 +size 1411415248 diff --git a/odxnffav/episode_rosbags/episode_3_2025-01-17-14-32-34.bag b/odxnffav/episode_rosbags/episode_3_2025-01-17-14-32-34.bag new file mode 100644 index 0000000000000000000000000000000000000000..758a44c5dd4c475450e45e19e0737eff455522fe --- /dev/null +++ b/odxnffav/episode_rosbags/episode_3_2025-01-17-14-32-34.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c179cf9f79290fd3e32ca26fa99bb4a3f9967555295fd562f2bc5e20ba2c014 +size 1412327663 diff --git a/odxnffav/episode_rosbags/episode_4_2025-01-17-14-33-16.bag b/odxnffav/episode_rosbags/episode_4_2025-01-17-14-33-16.bag new file mode 100644 index 0000000000000000000000000000000000000000..c075104b00b4ad0a6cb9ad326778cef4f110ceed --- /dev/null +++ b/odxnffav/episode_rosbags/episode_4_2025-01-17-14-33-16.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ead911d2453fca87d2f551836589bd4bb9223ac5c896906ec94bc56b39422ebb +size 1414301501 diff --git a/odxnffav/episode_rosbags/episode_8_2025-01-17-14-36-13.bag b/odxnffav/episode_rosbags/episode_8_2025-01-17-14-36-13.bag new file mode 100644 index 0000000000000000000000000000000000000000..ee8d05b78c92cde633595c36d5d5184ea3c47379 --- /dev/null +++ b/odxnffav/episode_rosbags/episode_8_2025-01-17-14-36-13.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fef30e72f1b2734ea9d1da6146481d9b8ddaf369e0bf623d5a243b95528ee70 +size 1413590467 diff --git a/odxnffav/eval_video/0_eval.mp4 b/odxnffav/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3dfe1848660789fc383b5946459acef2408c4562 --- /dev/null +++ b/odxnffav/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9913e877ba79f187dfcea6bc8c6ed0244e53fc12e8c981b56f34d8d880b539b2 +size 1170054 diff --git a/odxnffav/eval_video/10_eval.mp4 b/odxnffav/eval_video/10_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..407bae465b7861dfd9cc05bd9144cc6c6428c914 --- /dev/null +++ b/odxnffav/eval_video/10_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75a622b28aa17519a714919c9798b1c9050b1bb4fc26a3130abeebc87ebc7505 +size 1026281 diff --git a/odxnffav/eval_video/11_eval.mp4 b/odxnffav/eval_video/11_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..22e6679ee7bd01d1a482df7d9b4f38bce62d4edd --- /dev/null +++ b/odxnffav/eval_video/11_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5a9c02b219899b0e91253719aa3b9dc482a1dd2f12eaf4ca04061623f96ee24 +size 1391591 diff --git a/odxnffav/eval_video/12_eval.mp4 b/odxnffav/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a3c0b54e7054b122fe49d067d55c96ed68919638 --- /dev/null +++ b/odxnffav/eval_video/12_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c8ef51bd0e32f8f77f24d8b48fdd1666eb4d9882301298b0126ac7a064864fb +size 1395303 diff --git a/odxnffav/eval_video/13_eval.mp4 b/odxnffav/eval_video/13_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b499961ccf150aeb04d748014170007a00440c57 --- /dev/null +++ b/odxnffav/eval_video/13_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4585080ffb0d13dad79437b10dd80654dd17cb00fd5a556426aebfa73bbee7fe +size 1336285 diff --git a/odxnffav/eval_video/14_eval.mp4 b/odxnffav/eval_video/14_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..906ec6564e6207ab46f927e0b50c8c5d0082cc4f --- /dev/null +++ b/odxnffav/eval_video/14_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c74466fc3e808dda0cee5dd6e6b6933c46ce00bafeb05a8d87dac867ee1ad41 +size 1170418 diff --git a/odxnffav/eval_video/15_eval.mp4 b/odxnffav/eval_video/15_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1b3b07644ff13c8cf4028d0aad5ac920e9a5099b --- /dev/null +++ b/odxnffav/eval_video/15_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bb20ba2a7c41db7ced9b88ff5bdb4ca43f9a909439badf6036a7b55ad6bfbcf +size 1169456 diff --git a/odxnffav/eval_video/16_eval.mp4 b/odxnffav/eval_video/16_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..41922be0f00e539e8af93ec9859bfa9ff52f9040 --- /dev/null +++ b/odxnffav/eval_video/16_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f6b9b3e7e132b67ed2e8d594847d86167a8c1d6a02c0073ffd7646b2c677d48 +size 1479356 diff --git a/odxnffav/eval_video/17_eval.mp4 b/odxnffav/eval_video/17_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..25b665a1116442a9b8c8ca00bff9ac77eef73cb3 --- /dev/null +++ b/odxnffav/eval_video/17_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba7e994f26a18d93d39e0139da3e592d2c9474f5db0ff98722c00ca2a420b992 +size 1538746 diff --git a/odxnffav/eval_video/18_eval.mp4 b/odxnffav/eval_video/18_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ab1da036ba0047638183acfecd36a4c6d40dbbfe --- /dev/null +++ b/odxnffav/eval_video/18_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d49a7e0e6acc433fa848e00663c571fddae3753bd4d0aa2928a1e25fa15b417e +size 1512633 diff --git a/odxnffav/eval_video/19_eval.mp4 b/odxnffav/eval_video/19_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2c979e3eebbe378693e22bd76226191700c82c2b --- /dev/null +++ b/odxnffav/eval_video/19_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b74266e52a4c9294c18d2ce3b8d5b8ee0643533164bcdf7a75eb2967cd01b96c +size 1550606 diff --git a/odxnffav/eval_video/1_eval.mp4 b/odxnffav/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bd0fc43e8eed0f9845243e15cd2ac5cd967664dc --- /dev/null +++ b/odxnffav/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d7ecef067735039e088f167a382785ab1352bd14d5e981d38a68a82702a3db9 +size 1148915 diff --git a/odxnffav/eval_video/2_eval.mp4 b/odxnffav/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e4069b8d66ef5d350f43fdee68877db64cdb2737 --- /dev/null +++ b/odxnffav/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74f6d531a1bae5354927bf7ea063e3e1126bce72d9b353fef3f8fc6bee9101ea +size 1160047 diff --git a/odxnffav/eval_video/3_eval.mp4 b/odxnffav/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a70fce6f461a175c5b0d6a4a883f5ce68002580c --- /dev/null +++ b/odxnffav/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90efa260cbf26eb4711e3e033cbf6c0b19e468b211fb7ff8fea8018691d3eec4 +size 1189230 diff --git a/odxnffav/eval_video/4_eval.mp4 b/odxnffav/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..096edecb2b1c67000dc8da891e4414a048ecc2cc --- /dev/null +++ b/odxnffav/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87ec98a2ed597fa45d1d8ad4b64b196e737e9855db69fa9f6fe53b5970587bce +size 1212705 diff --git a/odxnffav/eval_video/5_eval.mp4 b/odxnffav/eval_video/5_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f1e9a273658a2147ecc28e2e7abf6ecaeaaf7244 --- /dev/null +++ b/odxnffav/eval_video/5_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5cc7767f4cc999092400e11bff56b409cf9c32f9254fc840838b0d77bd44557 +size 1146502 diff --git a/odxnffav/eval_video/6_eval.mp4 b/odxnffav/eval_video/6_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f8c545976dd5fc6c2f69468f61d6e4f218186c76 --- /dev/null +++ b/odxnffav/eval_video/6_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:507e5204685e3d527cd35f83a9075f64604ffb44391451e3c8914a36456ff99f +size 1162629 diff --git a/odxnffav/eval_video/7_eval.mp4 b/odxnffav/eval_video/7_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3b2b080549dea11bd48d59f731fe383ae73f99d1 --- /dev/null +++ b/odxnffav/eval_video/7_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6f988d87c4abf5cf1bd1dc749521fa6936d9b6af045c1f6735b4c695e3e3267 +size 1277174 diff --git a/odxnffav/eval_video/8_eval.mp4 b/odxnffav/eval_video/8_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..548806ae8a56529e7c3484e4b1f825804ee52706 --- /dev/null +++ b/odxnffav/eval_video/8_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c97055dba6783f8bd296241245f6665ca5c4bdf7eb55913ef15a600806612933 +size 1169733 diff --git a/odxnffav/eval_video/9_eval.mp4 b/odxnffav/eval_video/9_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b8850b87e2d72d57245fab801327e936c8d0c5a7 --- /dev/null +++ b/odxnffav/eval_video/9_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ad291b6cce26da44467c7d877f89a814593ebe7d4fdd6ab9f9195a372776382 +size 1180786 diff --git a/odxnffav/tb/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 b/odxnffav/tb/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 new file mode 100644 index 0000000000000000000000000000000000000000..741291f11eb95220fe17a9fa9c1df4b8c85fd4b7 --- /dev/null +++ b/odxnffav/tb/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:734c974afbeef73985955ab0db7299d292420136810c00beff1f063a4708a5bf +size 22784 diff --git a/odxnffav/wandb/debug-internal.log b/odxnffav/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a85bdc8e125d7055eca8d66b08f51bbf065f3dcb --- /dev/null +++ b/odxnffav/wandb/debug-internal.log @@ -0,0 +1,2125 @@ +2025-01-17 14:29:32,140 INFO StreamThr :2514330 [internal.py:wandb_internal():85] W&B internal server running at pid: 2514330, started at: 2025-01-17 14:29:32.140380 +2025-01-17 14:29:32,141 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status +2025-01-17 14:29:32,157 INFO WriterThread:2514330 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/run-odxnffav.wandb +2025-01-17 14:29:32,158 DEBUG SenderThread:2514330 [sender.py:send():379] send: header +2025-01-17 14:29:32,169 DEBUG SenderThread:2514330 [sender.py:send():379] send: run +2025-01-17 14:29:32,582 INFO SenderThread:2514330 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files +2025-01-17 14:29:32,583 INFO SenderThread:2514330 [sender.py:_start_run_threads():1188] run started: odxnffav with start time 1737142172.140455 +2025-01-17 14:29:32,600 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: check_version +2025-01-17 14:29:32,600 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: check_version +2025-01-17 14:29:32,642 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: run_start +2025-01-17 14:29:32,647 DEBUG HandlerThread:2514330 [system_info.py:__init__():26] System info init +2025-01-17 14:29:32,647 DEBUG HandlerThread:2514330 [system_info.py:__init__():41] System info init done +2025-01-17 14:29:32,647 INFO HandlerThread:2514330 [system_monitor.py:start():194] Starting system monitor +2025-01-17 14:29:32,647 INFO SystemMonitor:2514330 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-17 14:29:32,647 INFO HandlerThread:2514330 [system_monitor.py:probe():214] Collecting system info +2025-01-17 14:29:32,648 INFO SystemMonitor:2514330 [interfaces.py:start():188] Started cpu monitoring +2025-01-17 14:29:32,648 INFO SystemMonitor:2514330 [interfaces.py:start():188] Started disk monitoring +2025-01-17 14:29:32,649 INFO SystemMonitor:2514330 [interfaces.py:start():188] Started gpu monitoring +2025-01-17 14:29:32,650 INFO SystemMonitor:2514330 [interfaces.py:start():188] Started memory monitoring +2025-01-17 14:29:32,650 INFO SystemMonitor:2514330 [interfaces.py:start():188] Started network monitoring +2025-01-17 14:29:32,661 DEBUG HandlerThread:2514330 [system_info.py:probe():152] Probing system +2025-01-17 14:29:32,663 DEBUG HandlerThread:2514330 [system_info.py:_probe_git():137] Probing git +2025-01-17 14:29:32,667 DEBUG HandlerThread:2514330 [system_info.py:_probe_git():145] Probing git done +2025-01-17 14:29:32,667 DEBUG HandlerThread:2514330 [system_info.py:probe():200] Probing system done +2025-01-17 14:29:32,667 DEBUG HandlerThread:2514330 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-17T19:29:32.661542', 'startedAt': '2025-01-17T19:29:32.128161', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3563.8643749999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4501.083, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 374.1404724121094}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-17 14:29:32,667 INFO HandlerThread:2514330 [system_monitor.py:probe():224] Finished collecting system info +2025-01-17 14:29:32,668 INFO HandlerThread:2514330 [system_monitor.py:probe():227] Publishing system info +2025-01-17 14:29:32,668 DEBUG HandlerThread:2514330 [system_info.py:_save_code():44] Saving code +2025-01-17 14:29:32,673 DEBUG HandlerThread:2514330 [system_info.py:_save_code():67] Saving code done +2025-01-17 14:29:32,673 DEBUG HandlerThread:2514330 [system_info.py:_save_patches():84] Saving git patches +2025-01-17 14:29:32,768 DEBUG HandlerThread:2514330 [system_info.py:_save_patches():126] Saving git patches done +2025-01-17 14:29:32,769 INFO HandlerThread:2514330 [system_monitor.py:probe():229] Finished publishing system info +2025-01-17 14:29:32,772 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:29:32,772 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-17 14:29:32,772 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-17 14:29:32,772 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-17 14:29:32,773 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-17 14:29:33,000 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: python_packages +2025-01-17 14:29:33,001 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: python_packages +2025-01-17 14:29:33,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:29:33,001 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:29:33,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:33,005 INFO HandlerThread:2514330 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/tb" + save: true +} +control { + relay_id: "4d9c6c5a7c254e1f9a8ed9d676a8df15" +} +_info { + stream_id: "odxnffav" +} + +2025-01-17 14:29:33,005 DEBUG HandlerThread:2514330 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-17 14:29:33,020 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/bb0d3sle-wandb-metadata.json +2025-01-17 14:29:33,058 DEBUG SenderThread:2514330 [sender.py:send():379] send: telemetry +2025-01-17 14:29:33,058 DEBUG SenderThread:2514330 [sender.py:send():379] send: tbrecord +2025-01-17 14:29:33,059 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:29:33,059 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 with policy live +2025-01-17 14:29:33,067 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/yormr7cw-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 14:29:33,076 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/jftrgxs5-code/FISH/eval_robot.py +2025-01-17 14:29:33,079 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/dtth7ds4-diff.patch +2025-01-17 14:29:33,266 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/bpcho8yu-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:29:33,584 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 14:29:33,584 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/diff.patch +2025-01-17 14:29:33,584 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/requirements.txt +2025-01-17 14:29:33,584 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-metadata.json +2025-01-17 14:29:33,585 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:29:33,585 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/code/FISH/eval_robot.py +2025-01-17 14:29:33,585 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:29:33,585 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/code +2025-01-17 14:29:33,585 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/code/FISH +2025-01-17 14:29:34,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:35,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:35,583 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:29:36,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:37,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:38,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:29:38,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:39,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:40,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:40,915 DEBUG SenderThread:2514330 [sender.py:send():379] send: config +2025-01-17 14:29:41,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:42,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:43,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:43,585 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:29:44,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:29:44,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:45,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:46,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:47,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:47,587 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:29:48,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:29:48,001 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:29:48,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:49,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:49,445 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:29:49,587 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:29:50,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:51,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:52,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:53,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:54,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:55,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:29:55,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:56,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:57,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:58,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:59,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:00,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:00,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:01,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:02,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:03,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:30:03,001 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:30:03,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:04,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:05,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:05,160 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:05,592 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/config.yaml +2025-01-17 14:30:06,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:07,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:08,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:09,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:10,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:11,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:11,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:12,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:13,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:13,594 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:30:14,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:15,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:15,595 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:30:16,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:16,578 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:17,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:17,595 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:30:18,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:30:18,002 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:30:18,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:19,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:19,596 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:30:20,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:21,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:22,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:22,069 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:23,008 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:24,008 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:25,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:26,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:27,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:28,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:28,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:29,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:30,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:31,010 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:32,010 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:32,651 DEBUG SystemMonitor:2514330 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-17 14:30:32,651 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:30:33,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:30:33,004 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:30:33,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:33,140 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:34,010 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:35,010 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:36,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:37,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:38,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:39,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:39,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:40,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:41,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:42,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:43,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:44,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:44,313 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:44,513 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:30:44,513 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 with policy now +2025-01-17 14:30:44,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:30:44,514 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:30:44,514 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:30:44,516 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:30:44,603 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 +2025-01-17 14:30:44,603 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:30:44,603 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:30:44,603 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos +2025-01-17 14:30:44,603 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media +2025-01-17 14:30:44,750 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/5yktu7tk-media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 +2025-01-17 14:30:45,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:45,021 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/fb00d24w-media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 +2025-01-17 14:30:45,603 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:30:46,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:47,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:48,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:30:48,004 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:30:48,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:49,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:49,605 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:30:50,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:50,137 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:51,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:52,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:53,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:53,606 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:30:54,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:55,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:56,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:56,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:57,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:58,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:59,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:00,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:01,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:01,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:02,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:02,652 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:31:03,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:31:03,004 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:31:03,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:04,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:05,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:06,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:06,145 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:07,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:07,612 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:31:07,871 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/1xkuaz4f-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:31:08,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:09,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:09,613 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:10,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:11,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:11,614 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:11,849 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:12,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:13,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:13,615 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:14,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:15,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:16,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:17,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:17,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:18,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:31:18,004 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:31:18,049 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:19,016 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:20,016 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:21,016 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:22,016 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:22,081 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:23,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:24,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:25,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:25,618 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:26,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:27,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:27,621 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:28,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:29,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:30,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:31,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:31,620 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:32,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:32,652 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:31:32,653 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:33,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:31:33,004 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:31:33,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:34,020 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:35,021 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:36,021 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:37,021 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:38,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:38,021 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:38,618 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:31:38,618 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 with policy now +2025-01-17 14:31:38,619 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:31:38,619 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:31:38,619 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:31:38,621 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:31:38,623 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:31:38,623 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 +2025-01-17 14:31:38,623 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:31:38,870 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/9dxwxzrm-media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 +2025-01-17 14:31:39,022 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:39,148 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/b91yv1ob-media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 +2025-01-17 14:31:39,623 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:40,024 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:41,027 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:42,024 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:43,010 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:43,025 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:43,624 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:44,026 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:45,061 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:46,024 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:47,025 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:47,626 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:48,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:31:48,004 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:31:48,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:48,145 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:48,438 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:31:48,438 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:31:48,438 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:31:48,626 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:31:48,626 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:31:49,024 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:49,626 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:50,028 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:51,025 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:51,627 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:52,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:53,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:53,628 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:54,008 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:54,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:55,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:56,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:57,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:58,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:59,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:00,008 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:00,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:01,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:02,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:02,653 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:32:03,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:32:03,004 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:32:03,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:04,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:05,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:05,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:06,090 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:07,090 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:08,090 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:09,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:10,090 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:11,008 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:11,090 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:12,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:13,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:14,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:15,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:16,008 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:16,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:17,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:18,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:32:18,005 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:32:18,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:19,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:20,094 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:20,358 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:32:20,358 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 with policy now +2025-01-17 14:32:20,359 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:32:20,360 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:32:20,360 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:32:20,361 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:32:20,598 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/zetewlf8-media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 +2025-01-17 14:32:20,637 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:32:20,637 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 +2025-01-17 14:32:20,637 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:32:20,921 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/uhp1oif3-media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 +2025-01-17 14:32:21,099 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:21,387 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:21,637 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:32:22,101 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:22,638 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:32:23,104 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:24,105 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:25,107 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:25,639 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:32:26,104 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:26,881 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:27,102 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:28,109 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:29,106 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:29,640 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:32:30,107 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:31,106 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:31,275 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:32:31,275 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:32:31,276 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:32:31,641 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:32:31,642 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:32:31,934 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/5u52yykk-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:32:32,020 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:32,117 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:32,653 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:32:33,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:32:33,007 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:32:33,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:33,642 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:32:34,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:35,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:36,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:37,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:37,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:37,643 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:32:38,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:39,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:40,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:41,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:42,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:43,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:43,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:44,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:45,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:46,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:47,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:48,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:32:48,006 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:32:48,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:48,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:49,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:50,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:51,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:52,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:53,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:54,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:54,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:55,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:56,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:57,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:58,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:59,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:59,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:00,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:01,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:02,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:02,654 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:33:03,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:33:03,006 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:33:03,118 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:04,122 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:04,139 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:04,211 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:33:04,211 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 with policy now +2025-01-17 14:33:04,214 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:33:04,215 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:33:04,215 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:33:04,215 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:33:04,444 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/mqc8hnwd-media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 +2025-01-17 14:33:04,653 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:33:04,653 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 +2025-01-17 14:33:04,653 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:33:04,939 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/y1i3iiru-media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 +2025-01-17 14:33:05,120 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:05,653 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:06,122 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:06,653 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:07,123 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:08,129 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:09,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:09,654 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:10,025 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:10,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:11,125 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:11,655 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:12,127 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:13,129 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:13,327 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:33:13,327 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:33:13,328 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:33:13,656 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:33:13,656 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:33:14,133 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:15,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:15,185 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:15,657 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:16,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:17,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:18,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:33:18,013 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:33:18,133 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:19,130 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:19,658 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:20,130 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:20,213 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:21,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:21,659 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:22,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:23,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:23,659 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:24,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:25,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:25,661 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:25,805 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:26,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:27,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:28,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:29,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:29,662 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:30,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:31,018 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:31,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:32,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:32,654 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:33:33,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:33:33,007 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:33:33,133 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:34,133 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:35,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:36,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:36,154 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:37,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:38,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:39,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:40,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:41,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:42,019 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:42,135 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:43,135 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:44,135 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:45,135 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:46,146 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:46,522 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:33:46,522 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 with policy now +2025-01-17 14:33:46,523 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:33:46,523 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:33:46,523 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:33:46,525 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:33:46,668 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:33:46,668 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 +2025-01-17 14:33:46,668 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:33:46,772 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/e6lkw693-media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 +2025-01-17 14:33:47,039 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/vsihdu4v-media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 +2025-01-17 14:33:47,143 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:47,575 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:47,668 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:48,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:33:48,012 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:33:48,147 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:49,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:50,146 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:51,146 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:51,670 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:52,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:53,030 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:53,145 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:54,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:55,155 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:55,671 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:56,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:56,382 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:33:56,382 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:33:56,383 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:33:56,671 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:33:56,672 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:33:56,996 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/q7yso7ye-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:33:57,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:57,672 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:58,155 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:59,025 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:59,151 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:59,672 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:00,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:01,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:01,673 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:02,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:02,655 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:34:03,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:34:03,009 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:34:03,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:03,674 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:04,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:04,154 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:05,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:06,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:07,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:08,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:09,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:10,022 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:10,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:11,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:12,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:13,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:14,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:15,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:16,022 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:16,151 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:17,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:18,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:34:18,009 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:34:18,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:19,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:20,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:21,122 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:21,153 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:22,153 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:23,153 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:24,153 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:25,153 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:26,153 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:26,807 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:27,155 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:27,683 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:27,892 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:34:27,893 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 with policy now +2025-01-17 14:34:27,895 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:34:27,896 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:34:27,896 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:34:27,896 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:34:28,126 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/kzp17l1y-media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 +2025-01-17 14:34:28,156 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:28,683 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:34:28,683 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 +2025-01-17 14:34:28,683 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:34:28,964 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/lvqu7e2s-media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 +2025-01-17 14:34:29,165 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:29,683 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:30,179 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:31,174 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:32,175 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:32,613 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:32,655 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:34:33,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:34:33,014 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:34:33,176 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:33,685 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:34,189 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:35,193 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:35,685 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:36,186 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:37,200 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:38,024 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:38,191 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:39,195 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:40,199 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:40,687 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:34:41,200 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:41,439 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:34:41,439 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:34:41,440 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:34:41,688 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:34:41,688 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:41,688 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:34:42,193 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:43,037 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:43,196 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:43,688 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:44,189 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:45,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:45,689 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:46,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:47,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:48,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:34:48,009 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:34:48,168 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:48,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:49,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:50,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:51,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:52,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:53,189 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:53,692 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:54,023 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:54,189 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:55,189 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:55,693 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:56,189 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:57,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:57,694 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:58,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:59,024 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:59,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:59,694 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:00,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:01,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:01,695 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:02,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:02,655 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:35:03,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:35:03,010 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:35:03,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:03,696 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:04,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:04,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:05,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:06,191 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:07,191 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:07,697 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:08,191 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:09,191 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:09,698 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:10,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:10,191 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:11,191 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:11,699 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:12,194 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:12,891 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:35:12,891 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 with policy now +2025-01-17 14:35:12,900 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:35:12,901 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:35:12,901 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:35:12,901 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:35:13,121 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/hch8vyvc-media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 +2025-01-17 14:35:13,193 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:13,700 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:13,700 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:35:13,700 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 +2025-01-17 14:35:13,700 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:35:13,984 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/duthr6v8-media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 +2025-01-17 14:35:14,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:14,700 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:15,025 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:15,206 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:16,201 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:17,207 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:17,701 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:18,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:35:18,016 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:35:18,207 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:19,208 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:20,207 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:20,458 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:21,212 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:21,703 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:22,209 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:23,209 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:23,492 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:35:23,492 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:35:23,493 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:35:23,704 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:35:23,704 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:35:23,957 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/fa8ohva5-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:35:24,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:25,217 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:25,705 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:26,153 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:26,234 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:27,230 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:27,705 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:28,227 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:29,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:30,226 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:31,218 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:31,318 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:31,707 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:32,218 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:32,656 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:35:33,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:35:33,011 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:35:33,218 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:34,218 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:35,218 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:35,708 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:36,218 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:37,133 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:37,219 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:38,219 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:39,219 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:40,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:41,219 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:41,711 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:42,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:42,497 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:43,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:44,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:45,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:46,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:47,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:48,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:35:48,011 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:35:48,167 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:48,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:49,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:50,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:51,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:52,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:53,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:54,035 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:54,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:55,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:56,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:57,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:58,239 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:58,309 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:35:58,309 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:35:58,310 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 with policy now +2025-01-17 14:35:58,310 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:35:58,310 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:35:58,311 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:35:58,554 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/u9aybhrw-media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 +2025-01-17 14:35:58,717 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:35:58,717 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 +2025-01-17 14:35:58,717 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:35:58,998 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/lwdxb3bn-media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 +2025-01-17 14:35:59,238 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:59,334 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:59,717 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:00,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:00,718 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:01,240 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:02,250 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:02,656 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:36:03,026 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:36:03,026 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:36:03,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:03,719 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:04,249 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:04,882 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:05,246 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:05,719 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:06,250 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:07,262 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:08,262 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:09,254 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:09,721 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:36:10,263 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:10,551 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:36:10,551 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:36:10,551 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:10,551 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:36:10,721 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:36:11,265 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:11,722 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:12,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:13,259 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:14,252 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:15,254 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:15,723 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:15,957 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:16,253 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:17,253 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:18,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:36:18,013 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:36:18,254 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:19,254 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:20,255 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:21,163 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:21,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:22,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:23,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:24,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:25,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:26,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:27,042 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:27,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:28,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:29,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:30,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:31,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:32,042 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:32,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:32,657 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:36:33,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:36:33,014 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:36:33,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:34,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:35,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:36,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:37,133 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:37,258 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:38,258 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:39,258 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:40,258 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:41,268 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:41,733 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:42,142 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:42,269 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:42,292 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:36:42,292 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 with policy now +2025-01-17 14:36:42,333 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:36:42,333 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:36:42,334 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:36:42,334 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:36:42,518 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/b2xifp9z-media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 +2025-01-17 14:36:42,734 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:42,734 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:36:42,734 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 +2025-01-17 14:36:42,735 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:36:43,002 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/7tx1w5el-media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 +2025-01-17 14:36:43,272 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:43,734 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:44,272 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:44,734 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:45,269 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:46,272 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:47,278 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:47,735 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:48,023 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:36:48,023 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:36:48,081 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:48,275 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:49,271 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:49,736 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:50,278 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:51,278 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:51,600 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:36:51,601 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:36:51,601 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:36:51,737 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:36:51,737 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:36:52,001 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/jgu33evp-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:36:52,291 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:53,296 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:53,738 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:54,072 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:54,319 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:55,292 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:56,278 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:57,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:57,739 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:58,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:59,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:59,741 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:00,062 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:00,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:01,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:02,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:02,657 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:37:03,018 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:37:03,019 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:37:03,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:04,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:05,182 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:05,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:06,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:07,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:08,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:09,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:10,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:11,062 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:11,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:12,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:13,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:14,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:15,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:16,063 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:16,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:17,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:18,019 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:37:18,019 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:37:18,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:19,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:20,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:21,143 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:21,281 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:22,281 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:23,309 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:23,750 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:23,838 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:37:23,838 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 with policy now +2025-01-17 14:37:23,848 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:37:23,849 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:37:23,849 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:37:23,849 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:37:24,069 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/fdqkbph8-media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 +2025-01-17 14:37:24,288 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:24,750 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:24,750 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:37:24,750 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 +2025-01-17 14:37:24,750 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:37:25,051 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/cz2a2lup-media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 +2025-01-17 14:37:25,301 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:25,750 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:26,306 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:26,931 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:27,303 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:28,312 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:29,313 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:29,752 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:30,315 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:31,320 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:31,753 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:32,338 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:32,658 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:37:32,659 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:33,040 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:37:33,040 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:37:33,321 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:34,337 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:35,339 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:35,650 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:37:35,650 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:37:35,651 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:37:35,754 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:37:35,754 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:37:36,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:37,345 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:37,755 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:37,769 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:38,353 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:39,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:40,339 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:41,345 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:41,756 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:42,359 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:42,833 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:43,363 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:44,364 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:45,370 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:46,353 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:47,353 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:47,759 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:47,924 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:48,028 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:37:48,029 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:37:48,353 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:49,353 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:50,353 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:51,353 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:52,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:53,078 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:53,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:54,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:55,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:56,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:57,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:58,078 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:58,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:59,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:00,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:01,355 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:02,373 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:02,658 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:38:03,049 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:38:03,049 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:38:03,162 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:03,381 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:03,629 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:38:03,629 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 with policy now +2025-01-17 14:38:03,641 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:38:03,641 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:38:03,642 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:38:03,642 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:38:03,765 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:03,765 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:38:03,765 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 +2025-01-17 14:38:03,766 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:38:03,850 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/z9tzmpyw-media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 +2025-01-17 14:38:04,116 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/njlcov0v-media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 +2025-01-17 14:38:04,390 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:04,765 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:05,382 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:05,766 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:06,383 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:07,409 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:08,408 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:09,103 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:09,446 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:09,767 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:10,409 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:11,411 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:11,768 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:12,423 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:13,426 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:14,437 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:14,459 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:14,702 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:38:14,702 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:38:14,702 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:38:14,769 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:38:14,770 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:38:15,071 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/wbsxlg8g-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:38:15,447 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:15,770 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:16,412 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:17,426 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:17,771 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:18,075 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:38:18,075 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:38:18,413 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:19,431 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:19,707 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:19,771 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:20,435 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:21,438 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:22,438 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:23,439 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:24,438 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:24,733 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:25,426 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:25,774 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:26,426 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:27,426 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:28,426 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:29,426 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:30,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:30,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:31,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:32,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:32,659 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:38:33,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:38:33,045 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:38:33,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:34,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:35,126 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:35,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:36,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:37,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:38,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:39,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:40,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:41,090 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:41,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:42,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:43,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:44,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:45,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:46,090 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:46,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:47,429 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:48,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:38:48,045 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:38:48,429 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:49,429 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:50,429 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:51,163 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:51,429 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:52,459 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:53,312 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:38:53,313 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 with policy now +2025-01-17 14:38:53,321 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:38:53,321 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:38:53,322 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:38:53,322 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:38:53,441 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:53,554 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/6myrxrf9-media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 +2025-01-17 14:38:53,785 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:53,785 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:38:53,785 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 +2025-01-17 14:38:53,785 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:38:54,080 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/k8nb6l9z-media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 +2025-01-17 14:38:54,463 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:54,785 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:55,460 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:55,786 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:56,399 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:56,465 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:57,495 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:57,786 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:58,471 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:59,474 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:00,476 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:01,479 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:01,788 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:02,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:02,491 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:02,659 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:39:02,759 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:39:02,759 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:39:02,765 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:39:02,788 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:39:02,789 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:39:03,061 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:39:03,061 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:39:03,493 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:03,789 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:39:03,789 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:04,493 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:05,495 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:05,790 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:06,495 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:07,497 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:07,790 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:08,061 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:08,502 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:09,519 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:10,528 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:11,524 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:12,552 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:13,123 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:13,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:13,793 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:14,513 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:15,513 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:15,794 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:16,513 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:17,513 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:18,046 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:39:18,047 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:39:18,181 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:18,513 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:19,513 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:20,513 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:21,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:22,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:23,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:24,100 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:24,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:25,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:26,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:27,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:28,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:29,100 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:29,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:30,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:31,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:32,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:32,660 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:39:33,047 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:39:33,047 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:39:33,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:34,197 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:34,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:35,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:36,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:37,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:38,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:39,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:40,101 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:40,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:41,530 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:41,619 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:39:41,619 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 with policy now +2025-01-17 14:39:41,634 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:39:41,635 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:39:41,635 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:39:41,635 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:39:41,804 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:41,804 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:39:41,804 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 +2025-01-17 14:39:41,804 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:39:41,848 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/vmjffqzc-media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 +2025-01-17 14:39:42,122 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/ajxs2v80-media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 +2025-01-17 14:39:42,528 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:42,804 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:43,530 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:43,805 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:44,532 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:45,122 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:45,530 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:46,556 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:47,556 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:47,806 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:48,079 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:39:48,079 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:39:48,538 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:49,552 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:49,807 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:50,509 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:50,554 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:51,553 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:51,808 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:39:51,821 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:39:51,821 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:39:51,822 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:39:52,555 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:52,809 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:39:52,809 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:39:53,558 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:53,809 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:54,585 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:55,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:56,056 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:56,568 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:57,589 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:57,811 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:58,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:59,585 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:00,583 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:01,579 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:01,812 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:40:01,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:02,579 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:02,660 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:40:03,047 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:40:03,048 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:40:03,579 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:04,579 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:05,579 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:06,580 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:07,228 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:07,580 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:08,580 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:09,580 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:10,580 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:11,580 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:12,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:13,121 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:13,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:14,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:15,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:16,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:17,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:18,047 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:40:18,048 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:40:18,184 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:18,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:19,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:20,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:21,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:22,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:23,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:24,122 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:24,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:25,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:26,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:27,621 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:28,610 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:28,944 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:40:28,944 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 with policy now +2025-01-17 14:40:28,961 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:40:28,961 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:40:28,961 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:40:28,962 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:40:29,190 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/e51sumpa-media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 +2025-01-17 14:40:29,595 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:29,823 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:40:29,824 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:40:29,824 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 +2025-01-17 14:40:29,824 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:40:30,019 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:30,123 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/ikvanlqv-media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 +2025-01-17 14:40:30,601 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:30,824 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:40:31,615 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:32,615 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:32,661 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:40:33,077 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:40:33,077 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:40:33,636 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:33,825 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:40:34,616 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:35,203 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:35,648 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:36,649 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:37,649 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:37,827 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:40:38,635 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:38,827 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:40:38,880 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:40:38,881 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:40:38,881 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:40:39,088 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/df3m9gyi-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:40:39,647 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:39,828 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:40:39,828 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:40:40,284 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:40,643 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:41,638 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:42,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:43,621 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:43,829 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:40:44,621 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:45,621 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:46,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:46,621 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:47,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:48,050 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:40:48,050 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:40:48,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:49,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:50,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:51,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:52,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:52,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:53,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:54,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:55,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:56,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:57,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:57,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:58,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:59,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:00,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:01,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:02,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:02,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:02,661 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:41:03,050 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:41:03,050 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:41:03,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:04,624 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:05,624 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:06,624 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:07,624 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:08,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:08,624 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:09,645 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:10,628 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:10,929 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:41:10,929 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 with policy now +2025-01-17 14:41:10,959 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:41:10,960 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:41:10,960 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:41:10,960 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:41:11,157 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/0tso4q2r-media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 +2025-01-17 14:41:11,690 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:11,841 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:11,841 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:41:11,841 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 +2025-01-17 14:41:11,841 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:41:12,146 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/z4k9ieig-media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 +2025-01-17 14:41:12,681 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:12,841 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:13,177 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:13,649 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:14,640 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:15,665 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:15,843 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:16,662 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:17,657 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:18,060 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:41:18,060 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:41:18,206 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:18,682 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:19,668 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:19,844 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:20,686 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:20,845 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:41:20,933 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:41:20,934 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:41:20,934 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:41:21,685 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:21,845 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:21,845 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:41:22,696 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:23,699 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:23,846 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:24,178 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:24,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:25,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:25,847 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:26,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:27,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:28,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:29,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:30,174 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:30,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:31,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:32,661 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:41:32,675 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:33,050 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:41:33,051 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:41:33,675 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:34,675 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:35,174 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:35,675 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:36,675 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:37,675 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:38,675 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:39,676 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:40,174 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:40,676 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:41,676 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:42,676 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:43,677 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:44,677 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:45,677 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:46,174 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:46,677 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:47,677 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:48,051 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:41:48,051 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:41:48,677 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:49,677 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:50,678 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:51,174 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:51,678 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:52,691 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:52,819 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:41:52,819 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 with policy now +2025-01-17 14:41:52,860 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 +2025-01-17 14:41:52,861 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:41:52,861 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:41:52,861 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:41:52,861 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:41:52,862 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:41:53,046 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/w2acgh58-media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 +2025-01-17 14:41:53,359 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/r6js5gee-media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 +2025-01-17 14:41:53,694 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:53,861 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:53,861 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:41:54,687 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:55,719 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:56,187 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:56,693 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:57,714 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:57,862 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:58,714 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:59,717 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:00,713 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:01,723 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:01,864 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:01,901 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:02,662 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:42:02,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:03,082 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:42:03,083 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:42:03,753 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:04,748 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:05,769 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:06,748 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:06,866 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:42:06,990 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:42:06,990 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:42:06,990 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:06,990 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:42:07,772 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:07,866 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:07,867 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:42:08,771 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:09,777 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:09,867 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:10,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:11,776 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:11,868 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:12,786 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:12,920 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:13,775 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:14,802 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:15,789 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:16,776 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:17,787 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:18,071 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:42:18,071 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:42:18,162 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:18,789 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:19,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:19,872 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:20,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:21,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:21,873 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:22,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:23,187 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:23,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:24,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:25,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:26,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:27,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:28,187 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:28,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:29,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:30,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:31,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:32,662 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:42:32,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:33,054 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:42:33,054 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:42:33,208 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:33,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:33,878 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:34,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:35,780 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:36,780 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:37,780 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:38,780 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:39,187 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:39,782 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:40,781 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:41,781 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:42,781 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:43,782 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:44,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:44,782 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:45,782 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:46,808 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:47,786 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:47,884 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:47,973 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:42:47,973 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 with policy now +2025-01-17 14:42:47,981 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:42:47,981 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:42:47,981 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:42:47,982 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:42:48,059 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:42:48,059 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:42:48,200 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/8tzr7loo-media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 +2025-01-17 14:42:48,842 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:48,884 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:48,885 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:42:48,885 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 +2025-01-17 14:42:48,885 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:42:49,199 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/8tgwv9hd-media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 +2025-01-17 14:42:49,210 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:49,833 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:49,885 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:50,845 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:51,848 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:52,837 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:53,854 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:53,886 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:54,222 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:54,851 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:55,831 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:55,887 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:56,837 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:57,836 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:57,888 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:42:58,049 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:42:58,049 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:42:58,050 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:42:58,140 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/qc3rpfq7-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:42:58,831 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:58,889 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:42:59,878 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:59,881 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:59,889 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:00,842 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:01,848 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:02,663 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:43:02,865 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:03,077 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:43:03,077 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:43:03,851 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:03,891 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:04,861 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:05,201 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:05,841 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:06,860 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:07,878 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:08,902 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:09,880 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:10,211 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:10,875 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:11,866 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:12,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:13,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:13,895 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:14,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:15,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:16,200 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:16,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:17,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:18,056 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:43:18,056 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:43:18,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:19,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:20,868 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:21,200 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:21,868 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:22,868 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:23,868 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:24,868 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:25,869 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:26,869 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:27,200 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:27,869 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:28,869 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:29,869 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:30,869 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:31,870 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:32,200 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:32,663 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:43:32,870 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:33,056 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:43:33,057 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:43:33,870 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:34,870 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:35,871 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:36,871 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:37,201 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:37,871 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:38,871 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:39,906 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:40,644 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:43:40,644 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 with policy now +2025-01-17 14:43:40,649 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:43:40,650 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:43:40,650 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:43:40,737 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:43:40,884 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/r1qx0bkk-media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 +2025-01-17 14:43:40,901 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:40,907 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:40,907 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:43:40,907 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 +2025-01-17 14:43:40,908 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:43:41,179 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/0qjlko87-media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 +2025-01-17 14:43:41,900 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:41,908 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:42,810 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:42,908 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:42,933 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:43,895 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:43,908 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:44,910 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:45,913 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:46,910 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:46,937 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:47,919 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:48,056 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:43:48,057 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:43:48,146 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:48,936 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:49,958 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:50,911 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:50,993 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:51,111 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:43:51,111 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:43:51,111 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:43:51,912 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:43:51,912 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:43:51,959 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:52,912 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:52,989 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:53,226 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:53,970 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:54,913 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:55,019 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:55,957 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:56,960 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:57,962 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:58,915 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:59,000 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:59,211 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:00,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:01,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:01,994 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:02,664 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:44:03,021 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:03,078 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:44:03,079 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:44:03,991 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:04,229 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:05,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:05,985 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:06,918 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:06,985 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:07,985 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:08,985 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:09,986 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:10,206 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:10,986 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:11,986 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:12,986 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:13,986 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:14,986 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:15,206 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:15,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:16,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:17,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:18,058 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:44:18,058 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:44:18,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:19,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:20,206 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:20,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:21,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:22,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:23,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:24,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:25,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:26,206 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:26,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:27,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:28,988 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:29,988 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:30,988 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:31,207 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:31,988 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:32,664 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:44:32,996 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:33,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:44:33,131 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:44:33,696 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:44:33,696 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 with policy now +2025-01-17 14:44:33,711 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:44:33,711 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:44:33,712 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:44:33,719 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:44:33,915 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/a81qmh8j-media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 +2025-01-17 14:44:33,930 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:44:33,930 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 +2025-01-17 14:44:33,931 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:44:33,992 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:34,207 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/43vz04pg-media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 +2025-01-17 14:44:34,931 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:35,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:35,931 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:36,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:36,834 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:36,932 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:37,020 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:38,008 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:38,932 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:39,053 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:40,036 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:40,933 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:41,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:41,905 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:42,022 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:42,934 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:43,049 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:43,935 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:44:44,046 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:44,176 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:44:44,177 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:44:44,177 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:44:44,935 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:44,935 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:44:45,058 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:46,043 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:46,936 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:46,993 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:47,049 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:48,061 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:48,094 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:44:48,095 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:44:49,054 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:50,050 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:50,938 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:51,046 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:52,051 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:52,490 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:53,053 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:54,108 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:55,136 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:56,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:57,107 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:58,037 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:58,104 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:59,096 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:00,096 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:00,942 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:01,096 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:02,096 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:02,665 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:45:02,943 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:03,070 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:45:03,070 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:45:03,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:03,218 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:04,096 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:04,944 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:05,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:06,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:07,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:08,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:08,236 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:09,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:10,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:11,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:12,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:13,098 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:13,237 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:14,098 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:15,098 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:16,098 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:16,949 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:17,098 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:18,071 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:45:18,071 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:45:18,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:19,098 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:19,236 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:20,098 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:21,099 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:22,099 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:23,099 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:24,099 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:25,099 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:25,237 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:26,119 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:26,922 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:45:26,922 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 with policy now +2025-01-17 14:45:26,922 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:45:26,923 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:45:26,923 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:26,925 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:26,954 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:26,955 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:45:26,955 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 +2025-01-17 14:45:26,955 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:45:27,105 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:27,155 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/0ehdgalh-media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 +2025-01-17 14:45:27,434 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/mlf2r53b-media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 +2025-01-17 14:45:28,157 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:28,955 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:29,139 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:29,956 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:30,185 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:30,260 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:31,130 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:32,165 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:32,665 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:45:32,957 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:33,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:45:33,089 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:45:33,176 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:34,179 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:35,145 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:35,493 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:35,958 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:45:36,179 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:36,235 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:45:36,235 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:36,236 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:36,951 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-17 14:45:36,951 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: log_artifact +2025-01-17 14:45:36,959 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:36,959 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:45:36,959 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json +2025-01-17 14:45:36,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media +2025-01-17 14:45:36,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval +2025-01-17 14:45:36,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table +2025-01-17 14:45:37,110 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:45:37,147 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:37,148 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/myl9s5ei-media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json +2025-01-17 14:45:37,428 INFO wandb-upload_2:2514330 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpph57zsi3 +2025-01-17 14:45:37,482 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-17 14:45:37,497 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:45:37,810 INFO SenderThread:2514330 [sender.py:send_request_log_artifact():1525] logged artifact run-odxnffav-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0OTEwMzMyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MTU0NTk4OQ==', 'latestArtifact': None}} +2025-01-17 14:45:37,810 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:45:37,810 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json with policy now +2025-01-17 14:45:37,810 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:45:37,810 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,811 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,811 DEBUG SenderThread:2514330 [sender.py:send():379] send: telemetry +2025-01-17 14:45:37,811 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,815 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,815 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,816 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,816 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,816 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,816 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,817 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,817 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,817 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,817 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,818 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,818 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,818 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,818 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,819 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,819 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,819 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,820 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,820 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,820 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,820 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,821 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: log_artifact +2025-01-17 14:45:37,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:45:37,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:45:37,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json +2025-01-17 14:45:37,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval +2025-01-17 14:45:38,055 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/2nmrwxif-media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json +2025-01-17 14:45:38,148 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:38,189 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/dv04cvip-media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json +2025-01-17 14:45:38,223 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/p2c0lp91-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:45:38,234 INFO wandb-upload_2:2514330 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpl5qywjbs +2025-01-17 14:45:38,649 INFO SenderThread:2514330 [sender.py:send_request_log_artifact():1525] logged artifact run-odxnffav-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0OTEwMzM3Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MTU0NjAzMA==', 'latestArtifact': None}} +2025-01-17 14:45:38,649 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:45:38,649 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json with policy now +2025-01-17 14:45:38,649 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:45:38,650 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:38,650 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:38,904 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/aw88p4pp-media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json +2025-01-17 14:45:38,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:38,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:45:39,148 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:40,148 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:40,651 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:40,961 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:41,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:42,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:43,025 DEBUG SenderThread:2514330 [sender.py:send():379] send: exit +2025-01-17 14:45:43,025 INFO SenderThread:2514330 [sender.py:send_exit():586] handling exit code: 0 +2025-01-17 14:45:43,026 INFO SenderThread:2514330 [sender.py:send_exit():588] handling runtime: 970 +2025-01-17 14:45:43,026 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:43,026 INFO SenderThread:2514330 [sender.py:send_exit():594] send defer +2025-01-17 14:45:43,026 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:43,026 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-17 14:45:43,026 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:43,026 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-17 14:45:43,026 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 1 +2025-01-17 14:45:43,027 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:43,027 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-17 14:45:43,027 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:43,027 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-17 14:45:43,027 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 2 +2025-01-17 14:45:43,027 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:43,027 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-17 14:45:43,027 INFO HandlerThread:2514330 [system_monitor.py:finish():203] Stopping system monitor +2025-01-17 14:45:43,027 INFO HandlerThread:2514330 [interfaces.py:finish():200] Joined cpu monitor +2025-01-17 14:45:43,027 INFO HandlerThread:2514330 [interfaces.py:finish():200] Joined disk monitor +2025-01-17 14:45:43,028 DEBUG SystemMonitor:2514330 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-17 14:45:43,028 DEBUG SystemMonitor:2514330 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-17 14:45:43,079 INFO HandlerThread:2514330 [interfaces.py:finish():200] Joined gpu monitor +2025-01-17 14:45:43,079 INFO HandlerThread:2514330 [interfaces.py:finish():200] Joined memory monitor +2025-01-17 14:45:43,079 INFO HandlerThread:2514330 [interfaces.py:finish():200] Joined network monitor +2025-01-17 14:45:43,080 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:43,080 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-17 14:45:43,080 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 3 +2025-01-17 14:45:43,080 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:45:43,080 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:43,080 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-17 14:45:43,080 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:43,081 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-17 14:45:43,081 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 4 +2025-01-17 14:45:43,081 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:43,081 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-17 14:45:43,963 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:45:44,964 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:46,965 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/config.yaml +2025-01-17 14:45:49,242 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 14:45:49,243 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:49,243 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: keepalive +2025-01-17 14:45:49,243 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:49,244 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-17 14:45:49,244 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 5 +2025-01-17 14:45:49,244 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 14:45:49,244 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:49,244 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-17 14:45:49,244 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:45:49,245 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:49,246 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:49,246 DEBUG SenderThread:2514330 [sender.py:send():379] send: summary +2025-01-17 14:45:49,249 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:49,250 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:49,250 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-17 14:45:49,250 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 6 +2025-01-17 14:45:49,250 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:49,250 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-17 14:45:49,250 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:49,250 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-17 14:45:49,250 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 7 +2025-01-17 14:45:49,250 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:49,250 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:49,250 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-17 14:45:49,251 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:49,251 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-17 14:45:49,966 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:45:50,027 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 14:45:51,043 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 8 +2025-01-17 14:45:51,044 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 14:45:51,044 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:51,044 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-17 14:45:51,044 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:51,044 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-17 14:45:51,044 INFO SenderThread:2514330 [job_builder.py:build():440] Attempting to build job artifact +2025-01-17 14:45:51,047 INFO SenderThread:2514330 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-17 14:45:51,049 INFO SenderThread:2514330 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-17 14:45:51,068 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 9 +2025-01-17 14:45:51,068 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:51,068 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-17 14:45:51,068 DEBUG SenderThread:2514330 [sender.py:send():379] send: artifact +2025-01-17 14:45:51,751 INFO wandb-upload_2:2514330 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp5pworv_0 +2025-01-17 14:45:51,752 INFO wandb-upload_3:2514330 [upload_job.py:push():85] Skipped uploading /tmp/tmpflclq4of/wandb-job.json +2025-01-17 14:45:51,872 INFO wandb-upload_0:2514330 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpxz7r8926 +2025-01-17 14:45:51,967 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:52,027 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 14:45:52,466 INFO SenderThread:2514330 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ0OTEwNDY1OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ0NDE0MDQxMg==', 'versionIndex': 441}}} +2025-01-17 14:45:52,466 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:52,466 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-17 14:45:52,467 INFO SenderThread:2514330 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-17 14:45:52,968 INFO SenderThread:2514330 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files +2025-01-17 14:45:52,968 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/config.yaml config.yaml +2025-01-17 14:45:52,969 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json wandb-summary.json +2025-01-17 14:45:52,970 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/requirements.txt requirements.txt +2025-01-17 14:45:52,970 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-metadata.json wandb-metadata.json +2025-01-17 14:45:52,970 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:45:52,970 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 14:45:52,971 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log output.log +2025-01-17 14:45:52,971 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/diff.patch diff.patch +2025-01-17 14:45:52,971 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-17 14:45:52,975 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 +2025-01-17 14:45:52,975 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 +2025-01-17 14:45:52,975 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 +2025-01-17 14:45:52,975 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 +2025-01-17 14:45:52,975 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 +2025-01-17 14:45:52,975 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 +2025-01-17 14:45:52,975 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 +2025-01-17 14:45:52,977 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json +2025-01-17 14:45:52,977 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json +2025-01-17 14:45:52,977 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 10 +2025-01-17 14:45:52,977 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 14:45:52,977 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:52,977 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-17 14:45:52,977 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:52,977 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-17 14:45:52,977 INFO SenderThread:2514330 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 14:45:53,027 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 14:45:53,028 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 14:45:53,159 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/requirements.txt +2025-01-17 14:45:53,161 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:45:53,161 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/config.yaml +2025-01-17 14:45:53,228 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:53,428 INFO Thread-11 (_thread_body):2514330 [sender.py:transition_state():614] send defer: 11 +2025-01-17 14:45:53,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:53,429 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-17 14:45:53,429 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:53,429 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-17 14:45:53,429 INFO SenderThread:2514330 [file_pusher.py:join():175] waiting for file pusher +2025-01-17 14:45:53,434 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 12 +2025-01-17 14:45:53,434 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:53,434 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-17 14:45:53,434 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:53,434 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-17 14:45:53,435 INFO SenderThread:2514330 [file_stream.py:finish():601] file stream finish called +2025-01-17 14:45:53,699 INFO SenderThread:2514330 [file_stream.py:finish():605] file stream finish is done +2025-01-17 14:45:53,699 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 13 +2025-01-17 14:45:53,699 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:53,699 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-17 14:45:53,699 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:53,699 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-17 14:45:53,700 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 14 +2025-01-17 14:45:53,700 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:53,700 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-17 14:45:53,700 DEBUG SenderThread:2514330 [sender.py:send():379] send: final +2025-01-17 14:45:53,700 DEBUG SenderThread:2514330 [sender.py:send():379] send: footer +2025-01-17 14:45:53,700 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:53,700 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-17 14:45:53,701 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 14:45:53,701 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 14:45:53,702 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: server_info +2025-01-17 14:45:53,702 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 14:45:53,702 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: server_info +2025-01-17 14:45:53,702 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: get_summary +2025-01-17 14:45:53,704 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-17 14:45:53,705 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:53,766 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 14:45:53,766 INFO MainThread:2514330 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-17 14:45:53,767 INFO MainThread:2514330 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-17 14:45:53,767 INFO MainThread:2514330 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-17 14:45:53,767 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: shutdown +2025-01-17 14:45:53,767 INFO HandlerThread:2514330 [handler.py:finish():882] shutting down handler +2025-01-17 14:45:54,702 INFO WriterThread:2514330 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/run-odxnffav.wandb +2025-01-17 14:45:54,766 INFO SenderThread:2514330 [sender.py:finish():1615] shutting down sender +2025-01-17 14:45:54,766 INFO SenderThread:2514330 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 14:45:54,766 INFO SenderThread:2514330 [file_pusher.py:join():175] waiting for file pusher diff --git a/odxnffav/wandb/debug.log b/odxnffav/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..afb01a48fac18262258976b2e70104e18fb94caa --- /dev/null +++ b/odxnffav/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Configure stats pid to 2514148 +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/settings +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/logs/debug.log +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/logs/debug-internal.log +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_init.py:init():569] calling init triggers +2025-01-17 14:29:32,135 INFO MainThread:2514148 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3456_0', 'wandb_notes': '3456_0_req_3921_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav'} +2025-01-17 14:29:32,135 INFO MainThread:2514148 [wandb_init.py:init():619] starting backend +2025-01-17 14:29:32,135 INFO MainThread:2514148 [wandb_init.py:init():623] setting up manager +2025-01-17 14:29:32,139 INFO MainThread:2514148 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-17 14:29:32,140 INFO MainThread:2514148 [wandb_init.py:init():631] backend started and connected +2025-01-17 14:29:32,157 INFO MainThread:2514148 [wandb_init.py:init():720] updated telemetry +2025-01-17 14:29:32,166 INFO MainThread:2514148 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-17 14:29:32,600 INFO MainThread:2514148 [wandb_run.py:_on_init():2435] communicating current version +2025-01-17 14:29:32,638 INFO MainThread:2514148 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-17 14:29:32,638 INFO MainThread:2514148 [wandb_init.py:init():804] starting run threads in backend +2025-01-17 14:29:33,001 INFO MainThread:2514148 [wandb_run.py:_console_start():2413] atexit reg +2025-01-17 14:29:33,001 INFO MainThread:2514148 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-17 14:29:33,001 INFO MainThread:2514148 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-17 14:29:33,001 INFO MainThread:2514148 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-17 14:29:33,005 INFO MainThread:2514148 [wandb_init.py:init():847] run started, returning control to user process +2025-01-17 14:29:33,005 INFO MainThread:2514148 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/tb, True +2025-01-17 14:29:40,914 INFO MainThread:2514148 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-17 14:45:54,768 WARNING MsgRouterThr:2514148 [router.py:message_loop():77] message_loop has been closed diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/code/FISH/eval_robot.py b/odxnffav/wandb/run-20250117_142932-odxnffav/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/config.yaml b/odxnffav/wandb/run-20250117_142932-odxnffav/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ea69b02b1edfcd8d8511b979dcabb8e9069f5d69 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/config.yaml @@ -0,0 +1,927 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '3456_0' +wandb_notes: + desc: null + value: 3456_0_req_3921_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: null +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737142172 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/diff.patch b/odxnffav/wandb/run-20250117_142932-odxnffav/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..9505998782078316d897b259427f75a7fd990633 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/diff.patch @@ -0,0 +1,78 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0b377ae..a6b1973 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -132,8 +132,11 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++ + # RGBD+mask+act history +-wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..d9b15a5 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,12 +167,14 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ '3456_1', + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json new file mode 100644 index 0000000000000000000000000000000000000000..bf5505fbcae158b6a5dff0a3d108931093356226 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.95, 0.7177962362330672, 0.9987086634085542, 0.7564540355884719, 0.9934742729123935, 0.8371450527828175, 0.9942810795212667, 0.8401749133383372, 0.9943569618219861]]} \ No newline at end of file diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json new file mode 100644 index 0000000000000000000000000000000000000000..2b2b7549d6f9f22f9bef656b0b8722971b8d45a0 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[46, 60, 0.7666666666666667, 0.6276897489769186, 0.8759431391202536, 0.6518131270701051, 0.8551727830653333, 0.6771971508506194, 0.8416609502645478, 0.6822186559074268, 0.8325494527151673]]} \ No newline at end of file diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3dfe1848660789fc383b5946459acef2408c4562 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9913e877ba79f187dfcea6bc8c6ed0244e53fc12e8c981b56f34d8d880b539b2 +size 1170054 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..407bae465b7861dfd9cc05bd9144cc6c6428c914 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75a622b28aa17519a714919c9798b1c9050b1bb4fc26a3130abeebc87ebc7505 +size 1026281 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..22e6679ee7bd01d1a482df7d9b4f38bce62d4edd --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5a9c02b219899b0e91253719aa3b9dc482a1dd2f12eaf4ca04061623f96ee24 +size 1391591 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a3c0b54e7054b122fe49d067d55c96ed68919638 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c8ef51bd0e32f8f77f24d8b48fdd1666eb4d9882301298b0126ac7a064864fb +size 1395303 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b499961ccf150aeb04d748014170007a00440c57 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4585080ffb0d13dad79437b10dd80654dd17cb00fd5a556426aebfa73bbee7fe +size 1336285 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..906ec6564e6207ab46f927e0b50c8c5d0082cc4f --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c74466fc3e808dda0cee5dd6e6b6933c46ce00bafeb05a8d87dac867ee1ad41 +size 1170418 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1b3b07644ff13c8cf4028d0aad5ac920e9a5099b --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bb20ba2a7c41db7ced9b88ff5bdb4ca43f9a909439badf6036a7b55ad6bfbcf +size 1169456 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..41922be0f00e539e8af93ec9859bfa9ff52f9040 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f6b9b3e7e132b67ed2e8d594847d86167a8c1d6a02c0073ffd7646b2c677d48 +size 1479356 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..25b665a1116442a9b8c8ca00bff9ac77eef73cb3 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba7e994f26a18d93d39e0139da3e592d2c9474f5db0ff98722c00ca2a420b992 +size 1538746 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ab1da036ba0047638183acfecd36a4c6d40dbbfe --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d49a7e0e6acc433fa848e00663c571fddae3753bd4d0aa2928a1e25fa15b417e +size 1512633 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2c979e3eebbe378693e22bd76226191700c82c2b --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b74266e52a4c9294c18d2ce3b8d5b8ee0643533164bcdf7a75eb2967cd01b96c +size 1550606 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bd0fc43e8eed0f9845243e15cd2ac5cd967664dc --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d7ecef067735039e088f167a382785ab1352bd14d5e981d38a68a82702a3db9 +size 1148915 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e4069b8d66ef5d350f43fdee68877db64cdb2737 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74f6d531a1bae5354927bf7ea063e3e1126bce72d9b353fef3f8fc6bee9101ea +size 1160047 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a70fce6f461a175c5b0d6a4a883f5ce68002580c --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90efa260cbf26eb4711e3e033cbf6c0b19e468b211fb7ff8fea8018691d3eec4 +size 1189230 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..096edecb2b1c67000dc8da891e4414a048ecc2cc --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87ec98a2ed597fa45d1d8ad4b64b196e737e9855db69fa9f6fe53b5970587bce +size 1212705 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f1e9a273658a2147ecc28e2e7abf6ecaeaaf7244 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5cc7767f4cc999092400e11bff56b409cf9c32f9254fc840838b0d77bd44557 +size 1146502 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f8c545976dd5fc6c2f69468f61d6e4f218186c76 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:507e5204685e3d527cd35f83a9075f64604ffb44391451e3c8914a36456ff99f +size 1162629 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3b2b080549dea11bd48d59f731fe383ae73f99d1 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6f988d87c4abf5cf1bd1dc749521fa6936d9b6af045c1f6735b4c695e3e3267 +size 1277174 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..548806ae8a56529e7c3484e4b1f825804ee52706 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c97055dba6783f8bd296241245f6665ca5c4bdf7eb55913ef15a600806612933 +size 1169733 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b8850b87e2d72d57245fab801327e936c8d0c5a7 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ad291b6cce26da44467c7d877f89a814593ebe7d4fdd6ab9f9195a372776382 +size 1180786 diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log b/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..0d14dae47321e32c694a77a5d834d36dfbad5b33 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log @@ -0,0 +1,1105 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737142180.916284]: resetting environment +[INFO] [1737142180.920148]: cleared current plan +[INFO] [1737142180.920935]: moving to home +[INFO] [1737142181.923096]: reached home +[INFO] [1737142181.924700]: reset action history +[INFO] [1737142184.683864]: environment reset +Starting episode 0 +[INFO] [1737142184.688252]: resetting environment +[INFO] [1737142184.691453]: cleared current plan +[INFO] [1737142184.692302]: moving to home +[INFO] [1737142185.694472]: reached home +[INFO] [1737142185.695753]: reset action history +[INFO] [1737142188.404282]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1737142212.162541]: resetting environment +[INFO] [1737142212.165970]: cleared current plan +[INFO] [1737142212.167214]: moving to home +[INFO] [1737142213.869259]: reached home +[INFO] [1737142213.870831]: reset action history +[INFO] [1737142216.578221]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_0_2025-01-17-14-30-16.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:38151/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2514481] +process[print_text-2]: started with pid [2514482] +process[rosbag_record-3]: started with pid [2514483] +started bagging! +[INFO] [1737142243.587239]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737142244.313198]: Stored episode 1. +[INFO] [1737142244.313467]: Saving video... +[INFO] [1737142244.500464]: Video saved! +Starting episode 1 +[INFO] [1737142244.515611]: resetting environment +[INFO] [1737142244.520267]: cleared current plan +[INFO] [1737142244.520479]: moving to home +[INFO] [1737142248.122673]: reached home +[INFO] [1737142248.123825]: reset action history +[INFO] [1737142250.835523]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142266.903010]: resetting environment +[INFO] [1737142266.905384]: cleared current plan +[INFO] [1737142266.905585]: moving to home +[INFO] [1737142267.907185]: reached home +[INFO] [1737142267.908417]: reset action history +[INFO] [1737142270.617737]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_1_2025-01-17-14-31-10.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:46167/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2514594] +process[print_text-5]: started with pid [2514595] +process[rosbag_record-6]: started with pid [2514596] +started bagging! +For topic gripper_width: timestamp difference is -106849646 for nearest: 1737142283377292896 - target: 1737142283484142542 at idx 172 +For topic gripper_width: timestamp difference is -103665387 for nearest: 1737142288577293838 - target: 1737142288680959225 at idx 232 +[INFO] [1737142297.573980]: Storing episode... +[rosbag_record-6] killing on exit[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1737142298.335567]: Stored episode 2. +[INFO] [1737142298.336579]: Saving video... +[INFO] [1737142298.597755]: Video saved! +Starting episode 2 +[INFO] [1737142298.622518]: resetting environment +[INFO] [1737142298.630005]: cleared current plan +[INFO] [1737142298.632487]: moving to home +[INFO] [1737142302.240502]: reached home +[INFO] [1737142302.244981]: reset action history +[INFO] [1737142305.018389]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142308.045195]: resetting environment +[INFO] [1737142308.051219]: cleared current plan +[INFO] [1737142308.051705]: moving to home +[INFO] [1737142309.053213]: reached home +[INFO] [1737142309.053710]: reset action history +[INFO] [1737142311.760162]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_2_2025-01-17-14-31-51.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:43105/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2514765] +process[print_text-8]: started with pid [2514766] +process[rosbag_record-9]: started with pid [2514767] +started bagging! +[INFO] [1737142338.939306]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1737142339.745306]: Stored episode 3. +[INFO] [1737142339.751507]: Saving video... +[INFO] [1737142340.310915]: Video saved! +Starting episode 3 +[INFO] [1737142340.359904]: resetting environment +[INFO] [1737142340.386123]: cleared current plan +[INFO] [1737142340.386934]: moving to home +[WARN] [1737142340.295040]: Plan exhausted +[WARN] [1737142340.332885]: Plan exhausted +[WARN] [1737142340.372480]: Plan exhausted +[INFO] [1737142343.990539]: reached home +[INFO] [1737142343.993626]: reset action history +[INFO] [1737142346.880456]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142350.991918]: resetting environment +[INFO] [1737142351.011823]: cleared current plan +[INFO] [1737142351.016022]: moving to home +[INFO] [1737142352.019791]: reached home +[INFO] [1737142352.021736]: reset action history +[INFO] [1737142354.801163]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_3_2025-01-17-14-32-34.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:36091/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2514898] +process[print_text-11]: started with pid [2514899] +process[rosbag_record-12]: started with pid [2514900] +started bagging! +[INFO] [1737142382.794191]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit[print_text-11] killing on exit +[INFO] [1737142383.582154]: Stored episode 4. +[INFO] [1737142383.589453]: Saving video... +[INFO] [1737142384.156508]: Video saved! +Starting episode 4 +[INFO] [1737142384.217841]: resetting environment +[INFO] [1737142384.232284]: cleared current plan +[INFO] [1737142384.242103]: moving to home +[WARN] [1737142384.134609]: Plan exhausted +[WARN] [1737142384.172536]: Plan exhausted +[WARN] [1737142384.213089]: Plan exhausted +[INFO] [1737142387.856061]: reached home +[INFO] [1737142387.858516]: reset action history +[INFO] [1737142390.708878]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142393.118096]: resetting environment +[INFO] [1737142393.129758]: cleared current plan +[INFO] [1737142393.134448]: moving to home +[INFO] [1737142394.142017]: reached home +[INFO] [1737142394.147521]: reset action history +[INFO] [1737142396.938846]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_4_2025-01-17-14-33-16.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:46677/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2515044] +process[print_text-14]: started with pid [2515045] +process[rosbag_record-15]: started with pid [2515046] +started bagging! +For topic gripper_width: timestamp difference is -103426884 for nearest: 1737142399977291395 - target: 1737142400080718279 at idx 41 +For topic gripper_width: timestamp difference is -101792116 for nearest: 1737142400777293902 - target: 1737142400879086018 at idx 50 +For topic gripper_width: timestamp difference is -105052708 for nearest: 1737142403377294542 - target: 1737142403482347250 at idx 81 +For topic gripper_width: timestamp difference is -104097026 for nearest: 1737142404577293021 - target: 1737142404681390047 at idx 98 +For topic gripper_width: timestamp difference is -102166293 for nearest: 1737142406977293613 - target: 1737142407079459906 at idx 134 +For topic gripper_width: timestamp difference is -101534275 for nearest: 1737142407777291628 - target: 1737142407878825903 at idx 143 +[INFO] [1737142425.344437]: Storing episode... +[print_text-14] killing on exit +[pub_text-13] killing on exit +[rosbag_record-15] killing on exit +[INFO] [1737142426.152500]: Stored episode 5. +[INFO] [1737142426.161022]: Saving video... +[INFO] [1737142426.470166]: Video saved! +Starting episode 5 +[INFO] [1737142426.527910]: resetting environment +[INFO] [1737142426.559056]: cleared current plan +[INFO] [1737142426.570772]: moving to home +[INFO] [1737142430.184337]: reached home +[INFO] [1737142430.197806]: reset action history +[INFO] [1737142433.044936]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142435.784402]: resetting environment +[INFO] [1737142435.810566]: cleared current plan +[INFO] [1737142435.812850]: moving to home +[INFO] [1737142436.827258]: reached home +[INFO] [1737142436.836397]: reset action history +[INFO] [1737142439.641418]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_5_2025-01-17-14-33-59.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:42099/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-16]: started with pid [2515178] +process[print_text-17]: started with pid [2515179] +process[rosbag_record-18]: started with pid [2515180] +started bagging! +For topic gripper_width: timestamp difference is 125604277 for nearest: 1737142441443960553 - target: 1737142441318356276 at idx 30 +[INFO] [1737142466.807153]: Storing episode... +[print_text-17] killing on exit +[rosbag_record-18] killing on exit[pub_text-16] killing on exit +[INFO] [1737142467.587078]: Stored episode 6. +[INFO] [1737142467.589418]: Saving video... +[INFO] [1737142467.806311]: Video saved! +Starting episode 6 +[INFO] [1737142467.897191]: resetting environment +[INFO] [1737142467.937863]: cleared current plan +[INFO] [1737142467.941400]: moving to home +[INFO] [1737142471.558538]: reached home +[INFO] [1737142471.577541]: reset action history +[INFO] [1737142474.510766]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142480.715469]: resetting environment +[INFO] [1737142480.749579]: cleared current plan +[INFO] [1737142480.752646]: moving to home +[INFO] [1737142481.767734]: reached home +[INFO] [1737142481.776779]: reset action history +[INFO] [1737142484.568588]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_6_2025-01-17-14-34-44.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:32847/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-19]: started with pid [2515298] +process[print_text-20]: started with pid [2515299] +process[rosbag_record-21]: started with pid [2515300] +started bagging! +For topic gripper_width: timestamp difference is 100056886 for nearest: 1737142492743957519 - target: 1737142492643900633 at idx 120 +For topic gripper_width: timestamp difference is 100223620 for nearest: 1737142492943961222 - target: 1737142492843737602 at idx 122 +For topic gripper_width: timestamp difference is 102564050 for nearest: 1737142495743995143 - target: 1737142495641431093 at idx 160 +For topic gripper_width: timestamp difference is -163648743 for nearest: 1737142496077288728 - target: 1737142496240937471 at idx 164 +For topic gripper_width: timestamp difference is 103544294 for nearest: 1737142496943984090 - target: 1737142496840439796 at idx 171 +For topic gripper_width: timestamp difference is -159938098 for nearest: 1737142500477288960 - target: 1737142500637227058 at idx 215 +For topic gripper_width: timestamp difference is 107066067 for nearest: 1737142501143953236 - target: 1737142501036887169 at idx 220 +For topic gripper_width: timestamp difference is 107780824 for nearest: 1737142501943984638 - target: 1737142501836203814 at idx 229 +For topic gripper_width: timestamp difference is 110618911 for nearest: 1737142505343955598 - target: 1737142505233336687 at idx 249 +For topic gripper_width: timestamp difference is 110957006 for nearest: 1737142505743957141 - target: 1737142505633000135 at idx 249 +For topic gripper_width: timestamp difference is 111133741 for nearest: 1737142505943985628 - target: 1737142505832851887 at idx 249 +For topic gripper_width: timestamp difference is -154860683 for nearest: 1737142506677290015 - target: 1737142506832150698 at idx 249 +For topic gripper_width: timestamp difference is 113426392 for nearest: 1737142508943958943 - target: 1737142508830532551 at idx 249 +[INFO] [1737142511.522688]: Storing episode... +For topic gripper_width: timestamp difference is 115516091 for nearest: 1737142511543982173 - target: 1737142511428466082 at idx 249 +[rosbag_record-21] killing on exit +[pub_text-19] killing on exit +[print_text-20] killing on exit +[INFO] [1737142512.351356]: Stored episode 7. +[INFO] [1737142512.363157]: Saving video... +[INFO] [1737142512.821687]: Video saved! +Starting episode 7 +[INFO] [1737142512.908498]: resetting environment +[INFO] [1737142512.933249]: cleared current plan +[INFO] [1737142512.934911]: moving to home +[WARN] [1737142512.853372]: Plan exhausted +[WARN] [1737142512.894784]: Plan exhausted +[WARN] [1737142512.932470]: Plan exhausted +[INFO] [1737142516.542774]: reached home +[INFO] [1737142516.550921]: reset action history +[INFO] [1737142519.413335]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142523.061360]: resetting environment +[INFO] [1737142523.082246]: cleared current plan +[INFO] [1737142523.095479]: moving to home +[INFO] [1737142524.106604]: reached home +[INFO] [1737142524.108087]: reset action history +[INFO] [1737142526.905240]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_7_2025-01-17-14-35-26.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:44929/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-22]: started with pid [2515476] +process[print_text-23]: started with pid [2515477] +process[rosbag_record-24]: started with pid [2515478] +started bagging! +For topic gripper_width: timestamp difference is -102974976 for nearest: 1737142532877365266 - target: 1737142532980340242 at idx 81 +For topic gripper_width: timestamp difference is 101157125 for nearest: 1737142540443989452 - target: 1737142540342832327 at idx 177 +[INFO] [1737142557.016260]: Storing episode... +[rosbag_record-24] killing on exit +[print_text-23] killing on exit +[pub_text-22] killing on exit +[INFO] [1737142557.843175]: Stored episode 8. +[INFO] [1737142557.852946]: Saving video... +[INFO] [1737142558.198214]: Video saved! +Starting episode 8 +[INFO] [1737142558.282818]: resetting environment +[INFO] [1737142558.322479]: cleared current plan +[INFO] [1737142558.329183]: moving to home +[WARN] [1737142558.174189]: Plan exhausted +[WARN] [1737142558.215369]: Plan exhausted +[WARN] [1737142558.258652]: Plan exhausted +[WARN] [1737142558.295153]: Plan exhausted +[INFO] [1737142561.942728]: reached home +[INFO] [1737142561.948745]: reset action history +[INFO] [1737142564.879223]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142569.675499]: resetting environment +[INFO] [1737142569.700994]: cleared current plan +[INFO] [1737142569.705625]: moving to home +[INFO] [1737142570.713066]: reached home +[INFO] [1737142570.718347]: reset action history +[INFO] [1737142573.568481]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_8_2025-01-17-14-36-13.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:37463/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-25]: started with pid [2515607] +process[print_text-26]: started with pid [2515608] +process[rosbag_record-27]: started with pid [2515609] +started bagging! +[INFO] [1737142600.673674]: Storing episode... +[pub_text-25] killing on exit[print_text-26] killing on exit +[rosbag_record-27] killing on exit +[WARN] [1737142602.017694]: Plan exhausted +[INFO] [1737142601.491987]: Stored episode 9. +[INFO] [1737142601.501420]: Saving video... +[INFO] [1737142602.196101]: Video saved! +Starting episode 9 +[INFO] [1737142602.342405]: resetting environment +[INFO] [1737142602.363887]: cleared current plan +[INFO] [1737142602.373910]: moving to home +[WARN] [1737142602.053368]: Plan exhausted +[WARN] [1737142602.095833]: Plan exhausted +[WARN] [1737142602.133336]: Plan exhausted +[WARN] [1737142602.172951]: Plan exhausted +[WARN] [1737142602.215227]: Plan exhausted +[WARN] [1737142602.252872]: Plan exhausted +[WARN] [1737142602.292476]: Plan exhausted +[WARN] [1737142602.346611]: Plan exhausted +[INFO] [1737142606.088425]: reached home +[INFO] [1737142606.097269]: reset action history +[INFO] [1737142609.010161]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142611.406502]: resetting environment +[INFO] [1737142611.440965]: cleared current plan +[INFO] [1737142611.441188]: moving to home +[INFO] [1737142612.444992]: reached home +[INFO] [1737142612.456571]: reset action history +[INFO] [1737142615.325319]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_9_2025-01-17-14-36-55.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:33073/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-28]: started with pid [2515731] +process[print_text-29]: started with pid [2515732] +process[rosbag_record-30]: started with pid [2515756] +started bagging! +For topic gripper_width: timestamp difference is -138439518 for nearest: 1737142617277309555 - target: 1737142617415749073 at idx 36 +[INFO] [1737142642.283367]: Storing episode... +[print_text-29] killing on exit +[pub_text-28] killing on exit +[rosbag_record-30] killing on exit +[WARN] [1737142643.653341]: Plan exhausted +[WARN] [1737142643.702874]: Plan exhausted +[WARN] [1737142643.737624]: Plan exhausted +[WARN] [1737142643.773191]: Plan exhausted +[WARN] [1737142643.816596]: Plan exhausted +[WARN] [1737142643.854051]: Plan exhausted +[WARN] [1737142643.899692]: Plan exhausted +[INFO] [1737142643.146326]: Stored episode 10. +[INFO] [1737142643.160464]: Saving video... +[INFO] [1737142643.759160]: Video saved! +Starting episode 10 +[INFO] [1737142643.870480]: resetting environment +[INFO] [1737142643.920242]: cleared current plan +[INFO] [1737142643.929893]: moving to home +[INFO] [1737142647.548975]: reached home +[INFO] [1737142647.572056]: reset action history +[INFO] [1737142650.643844]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142655.679743]: resetting environment +[INFO] [1737142655.719046]: cleared current plan +[INFO] [1737142655.719314]: moving to home +[INFO] [1737142656.724928]: reached home +[INFO] [1737142656.731142]: reset action history +[INFO] [1737142659.788148]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_10_2025-01-17-14-37-39.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:36001/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-31]: started with pid [2515899] +process[print_text-32]: started with pid [2515900] +process[rosbag_record-33]: started with pid [2515901] +started bagging! +preempting episode +[INFO] [1737142681.848910]: Storing episode... +[rosbag_record-33] killing on exit +[print_text-32] killing on exit +[pub_text-31] killing on exit +[WARN] [1737142683.175978]: Plan exhausted +[WARN] [1737142683.213081]: Plan exhausted +[WARN] [1737142683.255951]: Plan exhausted +[WARN] [1737142683.297718]: Plan exhausted +[WARN] [1737142683.333148]: Plan exhausted +[WARN] [1737142683.374189]: Plan exhausted +[WARN] [1737142683.413227]: Plan exhausted +[WARN] [1737142683.455638]: Plan exhausted +[WARN] [1737142683.499601]: Plan exhausted +[WARN] [1737142683.537045]: Plan exhausted +[WARN] [1737142683.581765]: Plan exhausted +[WARN] [1737142683.616021]: Plan exhausted +[WARN] [1737142683.657650]: Plan exhausted +[WARN] [1737142683.697593]: Plan exhausted +[INFO] [1737142683.056390]: Stored episode 11. +[INFO] [1737142683.068631]: Saving video... +[INFO] [1737142683.493416]: Video saved! +Starting episode 11 +[INFO] [1737142683.649339]: resetting environment +[INFO] [1737142683.711258]: cleared current plan +[INFO] [1737142683.723615]: moving to home +[INFO] [1737142687.349556]: reached home +[INFO] [1737142687.375346]: reset action history +[INFO] [1737142690.359671]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142694.547654]: resetting environment +[INFO] [1737142694.599357]: cleared current plan +[INFO] [1737142694.605628]: moving to home +[INFO] [1737142695.621107]: reached home +[INFO] [1737142695.635198]: reset action history +[INFO] [1737142698.643180]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_11_2025-01-17-14-38-18.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:36653/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-34]: started with pid [2516043] +process[print_text-35]: started with pid [2516044] +process[rosbag_record-36]: started with pid [2516045] +started bagging! +[INFO] [1737142731.766319]: Storing episode... +[pub_text-34] killing on exit[print_text-35] killing on exit +[rosbag_record-36] killing on exit +[INFO] [1737142732.536617]: Stored episode 12. +[INFO] [1737142732.541456]: Saving video... +[WARN] [1737142733.133290]: Plan exhausted +[WARN] [1737142733.176512]: Plan exhausted +[WARN] [1737142733.216753]: Plan exhausted +[WARN] [1737142733.254257]: Plan exhausted +[WARN] [1737142733.293382]: Plan exhausted +[WARN] [1737142733.334032]: Plan exhausted +[WARN] [1737142733.372633]: Plan exhausted +[INFO] [1737142733.208305]: Video saved! +Starting episode 12 +[INFO] [1737142733.342296]: resetting environment +[INFO] [1737142733.376551]: cleared current plan +[INFO] [1737142733.398173]: moving to home +[INFO] [1737142737.006284]: reached home +[INFO] [1737142737.033780]: reset action history +[INFO] [1737142740.103122]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142742.838232]: resetting environment +[INFO] [1737142742.922225]: cleared current plan +[INFO] [1737142742.936002]: moving to home +[INFO] [1737142743.978325]: reached home +[INFO] [1737142743.992051]: reset action history +[INFO] [1737142747.000181]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_12_2025-01-17-14-39-07.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:42585/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-37]: started with pid [2516176] +process[print_text-38]: started with pid [2516177] +process[rosbag_record-39]: started with pid [2516178] +started bagging! +[INFO] [1737142780.266604]: Storing episode... +[rosbag_record-39] killing on exit[pub_text-37] killing on exit +[print_text-38] killing on exit +[INFO] [1737142781.122965]: Stored episode 13. +[INFO] [1737142781.144324]: Saving video... +[WARN] [1737142781.413913]: Plan exhausted +[WARN] [1737142781.458049]: Plan exhausted +[WARN] [1737142781.494759]: Plan exhausted +[WARN] [1737142781.533972]: Plan exhausted +[WARN] [1737142781.579556]: Plan exhausted +[WARN] [1737142781.612649]: Plan exhausted +[WARN] [1737142781.657290]: Plan exhausted +[INFO] [1737142781.429127]: Video saved! +Starting episode 13 +[INFO] [1737142781.644145]: resetting environment +[INFO] [1737142781.691581]: cleared current plan +[INFO] [1737142781.702781]: moving to home +[INFO] [1737142785.322795]: reached home +[INFO] [1737142785.343199]: reset action history +[INFO] [1737142788.453600]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142791.694153]: resetting environment +[INFO] [1737142791.756034]: cleared current plan +[INFO] [1737142791.787807]: moving to home +[INFO] [1737142792.823289]: reached home +[INFO] [1737142792.835250]: reset action history +[INFO] [1737142796.047330]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_13_2025-01-17-14-39-56.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:46215/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-40]: started with pid [2516322] +process[print_text-41]: started with pid [2516323] +process[rosbag_record-42]: started with pid [2516324] +started bagging! +[INFO] [1737142827.576968]: Storing episode... +[print_text-41] killing on exit +[rosbag_record-42] killing on exit +[pub_text-40] killing on exit +[INFO] [1737142828.434461]: Stored episode 14. +[INFO] [1737142828.444194]: Saving video... +[INFO] [1737142828.804122]: Video saved! +Starting episode 14 +[INFO] [1737142828.969907]: resetting environment +[INFO] [1737142829.011153]: cleared current plan +[INFO] [1737142829.015368]: moving to home +[WARN] [1737142828.732887]: Plan exhausted +[WARN] [1737142828.772492]: Plan exhausted +[WARN] [1737142828.836653]: Plan exhausted +[WARN] [1737142828.858561]: Plan exhausted +[WARN] [1737142828.920625]: Plan exhausted +[WARN] [1737142828.940298]: Plan exhausted +[WARN] [1737142828.985406]: Plan exhausted +[INFO] [1737142832.758991]: reached home +[INFO] [1737142832.790202]: reset action history +[INFO] [1737142835.972789]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142838.075832]: resetting environment +[INFO] [1737142838.153081]: cleared current plan +[INFO] [1737142838.155878]: moving to home +[INFO] [1737142839.187035]: reached home +[INFO] [1737142839.266135]: reset action history +[INFO] [1737142842.419759]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_14_2025-01-17-14-40-42.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:33621/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-43]: started with pid [2516465] +process[print_text-44]: started with pid [2516466] +process[rosbag_record-45]: started with pid [2516467] +started bagging! +[INFO] [1737142869.601729]: Storing episode... +[pub_text-43] killing on exit +[rosbag_record-45] killing on exit +[print_text-44] killing on exit +[INFO] [1737142870.462923]: Stored episode 15. +[INFO] [1737142870.484751]: Saving video... +[INFO] [1737142870.799479]: Video saved! +Starting episode 15 +[INFO] [1737142870.993051]: resetting environment +[INFO] [1737142871.072626]: cleared current plan +[INFO] [1737142871.114315]: moving to home +[WARN] [1737142870.972476]: Plan exhausted +[WARN] [1737142871.013148]: Plan exhausted +[WARN] [1737142871.053491]: Plan exhausted +[INFO] [1737142874.747177]: reached home +[INFO] [1737142874.756629]: reset action history +[INFO] [1737142877.866548]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142880.250030]: resetting environment +[INFO] [1737142880.323380]: cleared current plan +[INFO] [1737142880.370111]: moving to home +[INFO] [1737142881.398082]: reached home +[INFO] [1737142881.426512]: reset action history +[INFO] [1737142884.592843]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_15_2025-01-17-14-41-24.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:40397/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-46]: started with pid [2516609] +process[print_text-47]: started with pid [2516610] +process[rosbag_record-48]: started with pid [2516611] +started bagging! +[INFO] [1737142911.600403]: Storing episode... +[print_text-47] killing on exit +[rosbag_record-48] killing on exit[pub_text-46] killing on exit +[INFO] [1737142912.424966]: Stored episode 16. +[INFO] [1737142912.439868]: Saving video... +[INFO] [1737142912.670707]: Video saved! +Starting episode 16 +[INFO] [1737142912.857607]: resetting environment +[INFO] [1737142912.896440]: cleared current plan +[INFO] [1737142912.912337]: moving to home +[INFO] [1737142916.548372]: reached home +[INFO] [1737142916.559486]: reset action history +[INFO] [1737142919.850698]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142926.380488]: resetting environment +[INFO] [1737142926.518037]: cleared current plan +[INFO] [1737142926.534071]: moving to home +[INFO] [1737142927.549844]: reached home +[INFO] [1737142927.570662]: reset action history +[INFO] [1737142930.849732]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_16_2025-01-17-14-42-10.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:42571/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-49]: started with pid [2516761] +process[print_text-50]: started with pid [2516762] +process[rosbag_record-51]: started with pid [2516763] +started bagging! +For topic gripper_width: timestamp difference is 104603306 for nearest: 1737142952143957869 - target: 1737142952039354563 at idx 249 +For topic gripper_width: timestamp difference is 104763821 for nearest: 1737142952343953446 - target: 1737142952239189625 at idx 249 +For topic gripper_width: timestamp difference is 105251506 for nearest: 1737142952943955138 - target: 1737142952838703632 at idx 249 +[INFO] [1737142966.314300]: Storing episode... +[rosbag_record-51] killing on exit +[print_text-50] killing on exit +[pub_text-49] killing on exit +[INFO] [1737142967.015645]: Stored episode 17. +[INFO] [1737142967.046369]: Saving video... +[WARN] [1737142967.656300]: Plan exhausted +[WARN] [1737142967.701839]: Plan exhausted +[WARN] [1737142967.736776]: Plan exhausted +[WARN] [1737142967.780706]: Plan exhausted +[WARN] [1737142967.812629]: Plan exhausted +[WARN] [1737142967.857795]: Plan exhausted +[WARN] [1737142967.893630]: Plan exhausted +[WARN] [1737142967.933397]: Plan exhausted +[WARN] [1737142967.973284]: Plan exhausted +[WARN] [1737142968.013616]: Plan exhausted +[INFO] [1737142967.756012]: Video saved! +Starting episode 17 +[INFO] [1737142968.010382]: resetting environment +[INFO] [1737142968.030105]: cleared current plan +[INFO] [1737142968.037347]: moving to home +[INFO] [1737142971.662462]: reached home +[INFO] [1737142971.666549]: reset action history +[INFO] [1737142975.025438]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737142977.663164]: resetting environment +[INFO] [1737142977.786339]: cleared current plan +[INFO] [1737142977.802057]: moving to home +[INFO] [1737142978.832684]: reached home +[INFO] [1737142978.846706]: reset action history +[INFO] [1737142982.031124]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_17_2025-01-17-14-43-02.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:40967/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-52]: started with pid [2516894] +process[print_text-53]: started with pid [2516895] +process[rosbag_record-54]: started with pid [2516896] +started bagging! +[INFO] [1737143019.052129]: Storing episode... +[rosbag_record-54] killing on exit +[print_text-53] killing on exit +[pub_text-52] killing on exit +[WARN] [1737143020.212478]: Plan exhausted +[WARN] [1737143020.257479]: Plan exhausted +[WARN] [1737143020.295313]: Plan exhausted +[WARN] [1737143020.335148]: Plan exhausted +[WARN] [1737143020.372876]: Plan exhausted +[WARN] [1737143020.413511]: Plan exhausted +[WARN] [1737143020.462471]: Plan exhausted +[INFO] [1737143019.978747]: Stored episode 18. +[INFO] [1737143019.986694]: Saving video... +[INFO] [1737143020.509834]: Video saved! +Starting episode 18 +[INFO] [1737143020.721801]: resetting environment +[INFO] [1737143020.794696]: cleared current plan +[INFO] [1737143020.800176]: moving to home +[WARN] [1737143020.497908]: Plan exhausted +[WARN] [1737143020.532633]: Plan exhausted +[WARN] [1737143020.575478]: Plan exhausted +[WARN] [1737143020.615652]: Plan exhausted +[WARN] [1737143020.658404]: Plan exhausted +[WARN] [1737143020.706424]: Plan exhausted +[WARN] [1737143020.739338]: Plan exhausted +[WARN] [1737143020.776884]: Plan exhausted +[INFO] [1737143024.439208]: reached home +[INFO] [1737143024.462819]: reset action history +[INFO] [1737143027.632299]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143031.123159]: resetting environment +[INFO] [1737143031.226957]: cleared current plan +[INFO] [1737143031.253810]: moving to home +[INFO] [1737143032.269204]: reached home +[INFO] [1737143032.320950]: reset action history +[INFO] [1737143035.559535]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_18_2025-01-17-14-43-55.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:40601/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-55]: started with pid [2517039] +process[print_text-56]: started with pid [2517040] +process[rosbag_record-57]: started with pid [2517041] +started bagging! +[INFO] [1737143071.999195]: Storing episode... +[pub_text-55] killing on exit +[print_text-56] killing on exit[rosbag_record-57] killing on exit +[INFO] [1737143072.867610]: Stored episode 19. +[INFO] [1737143072.897307]: Saving video... +[INFO] [1737143073.524345]: Video saved! +[WARN] [1737143073.333484]: Plan exhausted +[WARN] [1737143073.375612]: Plan exhausted +[WARN] [1737143073.421117]: Plan exhausted +[WARN] [1737143073.452539]: Plan exhausted +[WARN] [1737143073.493712]: Plan exhausted +[WARN] [1737143073.536374]: Plan exhausted +[WARN] [1737143073.575983]: Plan exhausted +[WARN] [1737143073.630264]: Plan exhausted +[WARN] [1737143073.666907]: Plan exhausted +[WARN] [1737143073.719845]: Plan exhausted +[WARN] [1737143073.766875]: Plan exhausted +[WARN] [1737143073.788867]: Plan exhausted +Starting episode 19 +[INFO] [1737143073.756093]: resetting environment +[INFO] [1737143073.811643]: cleared current plan +[INFO] [1737143073.831628]: moving to home +[INFO] [1737143077.553211]: reached home +[INFO] [1737143077.571275]: reset action history +[INFO] [1737143080.855790]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143083.757488]: resetting environment +[INFO] [1737143083.885747]: cleared current plan +[INFO] [1737143083.898822]: moving to home +[INFO] [1737143084.945258]: reached home +[INFO] [1737143084.955273]: reset action history +[INFO] [1737143088.431620]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/episode_rosbags/episode_19_2025-01-17-14-44-48.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2514148.log +started roslaunch server http://158.130.50.37:35267/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-58]: started with pid [2517183] +process[print_text-59]: started with pid [2517184] +process[rosbag_record-60]: started with pid [2517185] +started bagging! +For topic gripper_width: timestamp difference is -115959953 for nearest: 1737143098777286221 - target: 1737143098893246174 at idx 151 +For topic gripper_width: timestamp difference is -110739313 for nearest: 1737143101377298750 - target: 1737143101488038063 at idx 185 +For topic gripper_width: timestamp difference is -106726858 for nearest: 1737143103377285031 - target: 1737143103484011889 at idx 210 +For topic gripper_width: timestamp difference is -107294862 for nearest: 1737143114577287610 - target: 1737143114684582472 at idx 249 +[INFO] [1737143125.345304]: Storing episode... +[pub_text-58] killing on exit +[print_text-59] killing on exit +[rosbag_record-60] killing on exit +[INFO] [1737143126.238214]: Stored episode 20. +[INFO] [1737143126.284782]: Saving video... +[INFO] [1737143126.736876]: Video saved! +[INFO] [1737143126.934518]: resetting environment +[INFO] [1737143126.970627]: cleared current plan +[INFO] [1737143126.970936]: moving to home +[WARN] [1737143126.693774]: Plan exhausted +[WARN] [1737143126.755199]: Plan exhausted +[WARN] [1737143126.815804]: Plan exhausted +[WARN] [1737143126.855570]: Plan exhausted +[WARN] [1737143126.890193]: Plan exhausted +[WARN] [1737143126.923737]: Plan exhausted +[WARN] [1737143126.956393]: Plan exhausted +[INFO] [1737143130.581871]: reached home +[INFO] [1737143130.593162]: reset action history +[INFO] [1737143133.980353]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/requirements.txt b/odxnffav/wandb/run-20250117_142932-odxnffav/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch b/odxnffav/wandb/run-20250117_142932-odxnffav/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch new file mode 100644 index 0000000000000000000000000000000000000000..ed960f9ae1689f11a78f267e474d398511995733 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch @@ -0,0 +1,95 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..a6b1973 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 5 ++num_eval: 20 + + debug_timestamps: False + open_loop: False +@@ -132,15 +132,18 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++ + # RGBD+mask+act history +-# wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..d9b15a5 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,12 +167,14 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ '3456_1', + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-metadata.json b/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9db86ac6fae95b48213461b0b79ef03d238ebdb --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-17T19:29:32.661542", + "startedAt": "2025-01-17T19:29:32.128161", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "ec42957dc10d08dbd8b134558fcd8f9e470ccc31" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3563.8643749999997, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.002, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4501.083, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 374.1404724121094 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json b/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..da3a3019d23f265f256413f6e367f3466f3b1a8f --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "9913e877ba79f187dfcea6bc8c6ed0244e53fc12e8c981b56f34d8d880b539b2", "size": 1170054, "path": "media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4"}, "global_step": 2602, "_timestamp": 1737143137.3998492, "_runtime": 965.2593941688538, "_step": 41, "eval/1_eval": {"_type": "video-file", "sha256": "4d7ecef067735039e088f167a382785ab1352bd14d5e981d38a68a82702a3db9", "size": 1148915, "path": "media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4"}, "eval/num_success": 19.0, "episode": 60.0, "eval/success_rate": 0.949999988079071, "eval/success_rate_lb_0.01": 0.7177962064743042, "eval/success_rate_ub_0.01": 0.9987086653709412, "eval/success_rate_lb_0.025": 0.7564540505409241, "eval/success_rate_ub_0.025": 0.9934742450714111, "eval/success_rate_lb_0.05": 0.8371450304985046, "eval/success_rate_ub_0.05": 0.9942810535430908, "eval/success_rate_lb_0.1": 0.8401749134063721, "eval/success_rate_ub_0.1": 0.9943569898605347, "eval/2_eval": {"_type": "video-file", "sha256": "74f6d531a1bae5354927bf7ea063e3e1126bce72d9b353fef3f8fc6bee9101ea", "size": 1160047, "path": "media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "90efa260cbf26eb4711e3e033cbf6c0b19e468b211fb7ff8fea8018691d3eec4", "size": 1189230, "path": "media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "87ec98a2ed597fa45d1d8ad4b64b196e737e9855db69fa9f6fe53b5970587bce", "size": 1212705, "path": "media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "b5cc7767f4cc999092400e11bff56b409cf9c32f9254fc840838b0d77bd44557", "size": 1146502, "path": "media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "507e5204685e3d527cd35f83a9075f64604ffb44391451e3c8914a36456ff99f", "size": 1162629, "path": "media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "b6f988d87c4abf5cf1bd1dc749521fa6936d9b6af045c1f6735b4c695e3e3267", "size": 1277174, "path": "media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "c97055dba6783f8bd296241245f6665ca5c4bdf7eb55913ef15a600806612933", "size": 1169733, "path": "media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "1ad291b6cce26da44467c7d877f89a814593ebe7d4fdd6ab9f9195a372776382", "size": 1180786, "path": "media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "75a622b28aa17519a714919c9798b1c9050b1bb4fc26a3130abeebc87ebc7505", "size": 1026281, "path": "media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "e5a9c02b219899b0e91253719aa3b9dc482a1dd2f12eaf4ca04061623f96ee24", "size": 1391591, "path": "media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "6c8ef51bd0e32f8f77f24d8b48fdd1666eb4d9882301298b0126ac7a064864fb", "size": 1395303, "path": "media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "4585080ffb0d13dad79437b10dd80654dd17cb00fd5a556426aebfa73bbee7fe", "size": 1336285, "path": "media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "9c74466fc3e808dda0cee5dd6e6b6933c46ce00bafeb05a8d87dac867ee1ad41", "size": 1170418, "path": "media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4"}, "eval/15_eval": {"_type": "video-file", "sha256": "8bb20ba2a7c41db7ced9b88ff5bdb4ca43f9a909439badf6036a7b55ad6bfbcf", "size": 1169456, "path": "media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4"}, "eval/16_eval": {"_type": "video-file", "sha256": "2f6b9b3e7e132b67ed2e8d594847d86167a8c1d6a02c0073ffd7646b2c677d48", "size": 1479356, "path": "media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4"}, "eval/17_eval": {"_type": "video-file", "sha256": "ba7e994f26a18d93d39e0139da3e592d2c9474f5db0ff98722c00ca2a420b992", "size": 1538746, "path": "media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4"}, "eval/18_eval": {"_type": "video-file", "sha256": "d49a7e0e6acc433fa848e00663c571fddae3753bd4d0aa2928a1e25fa15b417e", "size": 1512633, "path": "media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4"}, "eval/19_eval": {"_type": "video-file", "sha256": "b74266e52a4c9294c18d2ce3b8d5b8ee0643533164bcdf7a75eb2967cd01b96c", "size": 1550606, "path": "media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "52d7fd349d2c5dd77c8aa7aa34b8dba8a14ec3406e55fe78f4fae5efe359b619", "size": 399, "artifact_path": "wandb-client-artifact://dzh9j62wsz18ybvkravdboq0v5mzd9r18bftrn3tccbiciac1v862co6ayj9rk55eg6zph73xwz1ubzzoylyeqzdrguowxeavgncbxf5umq2lokwyrfk8gzvhpu5jbzy/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://eni2np04vhs6uxue74clou9y3kx7q77ppnfsz3lgr0r2ahz0dexuwczswbiyt550z80xnf9duqzbof8x1c6uz06t33saddbejwqqy2j66ncyzb5fzklkycudy8gw1gfl:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 46, "total_num_episodes": 60, "total_success_rate": 0.7666666666666667, "total_success_rate_lb_0.01": 0.6276897489769186, "total_success_rate_ub_0.01": 0.8759431391202536, "total_success_rate_lb_0.025": 0.6518131270701051, "total_success_rate_ub_0.025": 0.8551727830653333, "total_success_rate_lb_0.05": 0.6771971508506194, "total_success_rate_ub_0.05": 0.8416609502645478, "total_success_rate_lb_0.1": 0.6822186559074268, "total_success_rate_ub_0.1": 0.8325494527151673, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "4a20749ae6ca4d3c4e22a162eb46bb148b83a9f67645d6d263fb88bd22cf632a", "size": 520, "artifact_path": "wandb-client-artifact://p38b9albq1bl56epa97uwdsqgdyoifsx25xoo5qrwlvsy22q8julti16wdu4gza7ckitt8mky7vz3mukwc67kq9aphgwda84fkbwfli8lpnxsnqu7p204811yrnrsxt7/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://kyzmfi1z6nwskwhuszxfy80h4bwjcly8cwfk4opwppgjo5h7fikbcrbggfmajh3cl7xhu5x27rwc5zdnyi5azrit3nf3063i9w0m78dbmwgrpo9h1wxvecuiqmzcu57b:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 970}, "eval/total_success_rate": 0.7666666507720947, "eval/total_success_rate_lb_0.01": 0.6276897192001343, "eval/total_success_rate_ub_0.01": 0.875943124294281, "eval/total_success_rate_lb_0.025": 0.6518131494522095, "eval/total_success_rate_ub_0.025": 0.8551727533340454, "eval/total_success_rate_lb_0.05": 0.6771971583366394, "eval/total_success_rate_ub_0.05": 0.8416609764099121, "eval/total_success_rate_lb_0.1": 0.6822186708450317, "eval/total_success_rate_ub_0.1": 0.8325494527816772} \ No newline at end of file diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/logs/debug-internal.log b/odxnffav/wandb/run-20250117_142932-odxnffav/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a85bdc8e125d7055eca8d66b08f51bbf065f3dcb --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/logs/debug-internal.log @@ -0,0 +1,2125 @@ +2025-01-17 14:29:32,140 INFO StreamThr :2514330 [internal.py:wandb_internal():85] W&B internal server running at pid: 2514330, started at: 2025-01-17 14:29:32.140380 +2025-01-17 14:29:32,141 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status +2025-01-17 14:29:32,157 INFO WriterThread:2514330 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/run-odxnffav.wandb +2025-01-17 14:29:32,158 DEBUG SenderThread:2514330 [sender.py:send():379] send: header +2025-01-17 14:29:32,169 DEBUG SenderThread:2514330 [sender.py:send():379] send: run +2025-01-17 14:29:32,582 INFO SenderThread:2514330 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files +2025-01-17 14:29:32,583 INFO SenderThread:2514330 [sender.py:_start_run_threads():1188] run started: odxnffav with start time 1737142172.140455 +2025-01-17 14:29:32,600 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: check_version +2025-01-17 14:29:32,600 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: check_version +2025-01-17 14:29:32,642 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: run_start +2025-01-17 14:29:32,647 DEBUG HandlerThread:2514330 [system_info.py:__init__():26] System info init +2025-01-17 14:29:32,647 DEBUG HandlerThread:2514330 [system_info.py:__init__():41] System info init done +2025-01-17 14:29:32,647 INFO HandlerThread:2514330 [system_monitor.py:start():194] Starting system monitor +2025-01-17 14:29:32,647 INFO SystemMonitor:2514330 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-17 14:29:32,647 INFO HandlerThread:2514330 [system_monitor.py:probe():214] Collecting system info +2025-01-17 14:29:32,648 INFO SystemMonitor:2514330 [interfaces.py:start():188] Started cpu monitoring +2025-01-17 14:29:32,648 INFO SystemMonitor:2514330 [interfaces.py:start():188] Started disk monitoring +2025-01-17 14:29:32,649 INFO SystemMonitor:2514330 [interfaces.py:start():188] Started gpu monitoring +2025-01-17 14:29:32,650 INFO SystemMonitor:2514330 [interfaces.py:start():188] Started memory monitoring +2025-01-17 14:29:32,650 INFO SystemMonitor:2514330 [interfaces.py:start():188] Started network monitoring +2025-01-17 14:29:32,661 DEBUG HandlerThread:2514330 [system_info.py:probe():152] Probing system +2025-01-17 14:29:32,663 DEBUG HandlerThread:2514330 [system_info.py:_probe_git():137] Probing git +2025-01-17 14:29:32,667 DEBUG HandlerThread:2514330 [system_info.py:_probe_git():145] Probing git done +2025-01-17 14:29:32,667 DEBUG HandlerThread:2514330 [system_info.py:probe():200] Probing system done +2025-01-17 14:29:32,667 DEBUG HandlerThread:2514330 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-17T19:29:32.661542', 'startedAt': '2025-01-17T19:29:32.128161', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3563.8643749999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4501.083, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 374.1404724121094}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-17 14:29:32,667 INFO HandlerThread:2514330 [system_monitor.py:probe():224] Finished collecting system info +2025-01-17 14:29:32,668 INFO HandlerThread:2514330 [system_monitor.py:probe():227] Publishing system info +2025-01-17 14:29:32,668 DEBUG HandlerThread:2514330 [system_info.py:_save_code():44] Saving code +2025-01-17 14:29:32,673 DEBUG HandlerThread:2514330 [system_info.py:_save_code():67] Saving code done +2025-01-17 14:29:32,673 DEBUG HandlerThread:2514330 [system_info.py:_save_patches():84] Saving git patches +2025-01-17 14:29:32,768 DEBUG HandlerThread:2514330 [system_info.py:_save_patches():126] Saving git patches done +2025-01-17 14:29:32,769 INFO HandlerThread:2514330 [system_monitor.py:probe():229] Finished publishing system info +2025-01-17 14:29:32,772 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:29:32,772 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-17 14:29:32,772 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-17 14:29:32,772 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-17 14:29:32,773 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-17 14:29:33,000 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: python_packages +2025-01-17 14:29:33,001 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: python_packages +2025-01-17 14:29:33,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:29:33,001 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:29:33,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:33,005 INFO HandlerThread:2514330 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/tb" + save: true +} +control { + relay_id: "4d9c6c5a7c254e1f9a8ed9d676a8df15" +} +_info { + stream_id: "odxnffav" +} + +2025-01-17 14:29:33,005 DEBUG HandlerThread:2514330 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-17 14:29:33,020 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/bb0d3sle-wandb-metadata.json +2025-01-17 14:29:33,058 DEBUG SenderThread:2514330 [sender.py:send():379] send: telemetry +2025-01-17 14:29:33,058 DEBUG SenderThread:2514330 [sender.py:send():379] send: tbrecord +2025-01-17 14:29:33,059 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:29:33,059 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 with policy live +2025-01-17 14:29:33,067 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/yormr7cw-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 14:29:33,076 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/jftrgxs5-code/FISH/eval_robot.py +2025-01-17 14:29:33,079 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/dtth7ds4-diff.patch +2025-01-17 14:29:33,266 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/bpcho8yu-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:29:33,584 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 14:29:33,584 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/diff.patch +2025-01-17 14:29:33,584 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/requirements.txt +2025-01-17 14:29:33,584 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-metadata.json +2025-01-17 14:29:33,585 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:29:33,585 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/code/FISH/eval_robot.py +2025-01-17 14:29:33,585 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:29:33,585 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/code +2025-01-17 14:29:33,585 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/code/FISH +2025-01-17 14:29:34,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:35,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:35,583 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:29:36,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:37,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:38,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:29:38,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:39,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:40,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:40,915 DEBUG SenderThread:2514330 [sender.py:send():379] send: config +2025-01-17 14:29:41,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:42,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:43,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:43,585 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:29:44,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:29:44,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:45,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:46,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:47,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:47,587 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:29:48,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:29:48,001 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:29:48,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:49,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:49,445 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:29:49,587 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:29:50,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:51,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:52,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:53,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:54,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:55,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:29:55,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:56,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:57,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:58,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:29:59,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:00,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:00,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:01,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:02,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:03,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:30:03,001 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:30:03,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:04,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:05,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:05,160 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:05,592 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/config.yaml +2025-01-17 14:30:06,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:07,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:08,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:09,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:10,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:11,002 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:11,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:12,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:13,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:13,594 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:30:14,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:15,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:15,595 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:30:16,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:16,578 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:17,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:17,595 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:30:18,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:30:18,002 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:30:18,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:19,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:19,596 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:30:20,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:21,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:22,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:22,069 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:23,008 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:24,008 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:25,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:26,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:27,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:28,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:28,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:29,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:30,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:31,010 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:32,010 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:32,651 DEBUG SystemMonitor:2514330 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-17 14:30:32,651 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:30:33,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:30:33,004 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:30:33,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:33,140 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:34,010 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:35,010 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:36,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:37,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:38,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:39,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:39,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:40,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:41,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:42,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:43,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:44,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:44,313 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:44,513 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:30:44,513 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 with policy now +2025-01-17 14:30:44,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:30:44,514 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:30:44,514 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:30:44,516 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:30:44,603 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 +2025-01-17 14:30:44,603 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:30:44,603 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:30:44,603 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos +2025-01-17 14:30:44,603 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media +2025-01-17 14:30:44,750 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/5yktu7tk-media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 +2025-01-17 14:30:45,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:45,021 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/fb00d24w-media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 +2025-01-17 14:30:45,603 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:30:46,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:47,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:48,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:30:48,004 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:30:48,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:49,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:49,605 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:30:50,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:50,137 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:51,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:52,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:53,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:53,606 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:30:54,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:55,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:56,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:30:56,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:57,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:58,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:30:59,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:00,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:01,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:01,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:02,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:02,652 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:31:03,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:31:03,004 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:31:03,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:04,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:05,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:06,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:06,145 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:07,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:07,612 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:31:07,871 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/1xkuaz4f-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:31:08,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:09,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:09,613 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:10,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:11,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:11,614 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:11,849 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:12,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:13,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:13,615 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:14,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:15,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:16,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:17,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:17,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:18,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:31:18,004 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:31:18,049 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:19,016 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:20,016 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:21,016 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:22,016 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:22,081 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:23,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:24,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:25,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:25,618 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:26,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:27,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:27,621 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:28,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:29,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:30,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:31,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:31,620 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:32,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:32,652 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:31:32,653 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:33,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:31:33,004 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:31:33,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:34,020 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:35,021 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:36,021 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:37,021 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:38,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:38,021 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:38,618 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:31:38,618 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 with policy now +2025-01-17 14:31:38,619 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:31:38,619 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:31:38,619 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:31:38,621 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:31:38,623 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:31:38,623 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 +2025-01-17 14:31:38,623 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:31:38,870 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/9dxwxzrm-media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 +2025-01-17 14:31:39,022 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:39,148 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/b91yv1ob-media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 +2025-01-17 14:31:39,623 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:40,024 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:41,027 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:42,024 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:43,010 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:43,025 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:43,624 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:44,026 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:45,061 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:46,024 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:47,025 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:47,626 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:48,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:31:48,004 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:31:48,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:48,145 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:48,438 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:31:48,438 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:31:48,438 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:31:48,626 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:31:48,626 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:31:49,024 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:49,626 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:50,028 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:51,025 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:51,627 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:52,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:53,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:53,628 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:31:54,008 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:31:54,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:55,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:56,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:57,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:58,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:31:59,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:00,008 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:00,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:01,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:02,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:02,653 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:32:03,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:32:03,004 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:32:03,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:04,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:05,088 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:05,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:06,090 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:07,090 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:08,090 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:09,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:10,090 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:11,008 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:11,090 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:12,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:13,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:14,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:15,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:16,008 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:16,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:17,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:18,004 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:32:18,005 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:32:18,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:19,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:20,094 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:20,358 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:32:20,358 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 with policy now +2025-01-17 14:32:20,359 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:32:20,360 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:32:20,360 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:32:20,361 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:32:20,598 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/zetewlf8-media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 +2025-01-17 14:32:20,637 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:32:20,637 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 +2025-01-17 14:32:20,637 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:32:20,921 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/uhp1oif3-media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 +2025-01-17 14:32:21,099 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:21,387 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:21,637 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:32:22,101 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:22,638 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:32:23,104 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:24,105 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:25,107 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:25,639 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:32:26,104 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:26,881 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:27,102 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:28,109 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:29,106 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:29,640 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:32:30,107 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:31,106 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:31,275 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:32:31,275 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:32:31,276 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:32:31,641 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:32:31,642 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:32:31,934 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/5u52yykk-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:32:32,020 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:32,117 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:32,653 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:32:33,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:32:33,007 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:32:33,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:33,642 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:32:34,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:35,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:36,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:37,091 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:37,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:37,643 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:32:38,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:39,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:40,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:41,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:42,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:43,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:43,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:44,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:45,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:46,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:47,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:48,006 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:32:48,006 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:32:48,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:48,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:49,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:50,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:51,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:52,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:53,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:54,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:54,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:55,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:56,112 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:57,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:58,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:32:59,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:32:59,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:00,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:01,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:02,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:02,654 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:33:03,005 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:33:03,006 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:33:03,118 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:04,122 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:04,139 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:04,211 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:33:04,211 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 with policy now +2025-01-17 14:33:04,214 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:33:04,215 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:33:04,215 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:33:04,215 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:33:04,444 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/mqc8hnwd-media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 +2025-01-17 14:33:04,653 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:33:04,653 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 +2025-01-17 14:33:04,653 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:33:04,939 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/y1i3iiru-media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 +2025-01-17 14:33:05,120 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:05,653 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:06,122 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:06,653 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:07,123 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:08,129 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:09,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:09,654 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:10,025 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:10,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:11,125 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:11,655 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:12,127 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:13,129 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:13,327 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:33:13,327 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:33:13,328 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:33:13,656 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:33:13,656 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:33:14,133 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:15,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:15,185 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:15,657 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:16,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:17,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:18,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:33:18,013 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:33:18,133 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:19,130 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:19,658 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:20,130 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:20,213 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:21,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:21,659 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:22,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:23,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:23,659 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:24,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:25,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:25,661 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:25,805 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:26,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:27,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:28,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:29,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:29,662 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:30,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:31,018 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:31,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:32,132 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:32,654 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:33:33,007 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:33:33,007 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:33:33,133 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:34,133 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:35,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:36,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:36,154 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:37,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:38,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:39,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:40,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:41,134 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:42,019 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:42,135 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:43,135 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:44,135 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:45,135 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:46,146 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:46,522 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:33:46,522 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 with policy now +2025-01-17 14:33:46,523 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:33:46,523 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:33:46,523 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:33:46,525 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:33:46,668 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:33:46,668 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 +2025-01-17 14:33:46,668 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:33:46,772 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/e6lkw693-media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 +2025-01-17 14:33:47,039 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/vsihdu4v-media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 +2025-01-17 14:33:47,143 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:47,575 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:47,668 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:48,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:33:48,012 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:33:48,147 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:49,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:50,146 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:51,146 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:51,670 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:52,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:53,030 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:53,145 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:54,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:55,155 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:55,671 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:56,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:56,382 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:33:56,382 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:33:56,383 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:33:56,671 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:33:56,672 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:33:56,996 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/q7yso7ye-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:33:57,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:57,672 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:33:58,155 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:59,025 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:33:59,151 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:33:59,672 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:00,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:01,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:01,673 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:02,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:02,655 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:34:03,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:34:03,009 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:34:03,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:03,674 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:04,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:04,154 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:05,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:06,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:07,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:08,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:09,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:10,022 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:10,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:11,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:12,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:13,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:14,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:15,150 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:16,022 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:16,151 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:17,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:18,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:34:18,009 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:34:18,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:19,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:20,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:21,122 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:21,153 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:22,153 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:23,153 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:24,153 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:25,153 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:26,153 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:26,807 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:27,155 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:27,683 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:27,892 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:34:27,893 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 with policy now +2025-01-17 14:34:27,895 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:34:27,896 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:34:27,896 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:34:27,896 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:34:28,126 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/kzp17l1y-media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 +2025-01-17 14:34:28,156 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:28,683 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:34:28,683 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 +2025-01-17 14:34:28,683 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:34:28,964 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/lvqu7e2s-media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 +2025-01-17 14:34:29,165 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:29,683 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:30,179 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:31,174 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:32,175 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:32,613 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:32,655 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:34:33,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:34:33,014 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:34:33,176 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:33,685 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:34,189 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:35,193 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:35,685 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:36,186 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:37,200 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:38,024 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:38,191 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:39,195 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:40,199 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:40,687 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:34:41,200 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:41,439 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:34:41,439 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:34:41,440 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:34:41,688 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:34:41,688 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:41,688 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:34:42,193 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:43,037 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:43,196 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:43,688 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:44,189 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:45,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:45,689 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:46,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:47,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:48,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:34:48,009 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:34:48,168 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:48,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:49,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:50,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:51,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:52,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:53,189 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:53,692 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:54,023 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:54,189 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:55,189 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:55,693 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:56,189 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:57,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:57,694 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:34:58,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:59,024 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:34:59,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:34:59,694 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:00,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:01,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:01,695 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:02,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:02,655 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:35:03,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:35:03,010 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:35:03,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:03,696 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:04,111 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:04,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:05,190 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:06,191 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:07,191 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:07,697 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:08,191 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:09,191 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:09,698 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:10,001 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:10,191 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:11,191 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:11,699 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:12,194 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:12,891 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:35:12,891 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 with policy now +2025-01-17 14:35:12,900 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:35:12,901 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:35:12,901 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:35:12,901 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:35:13,121 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/hch8vyvc-media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 +2025-01-17 14:35:13,193 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:13,700 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:13,700 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:35:13,700 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 +2025-01-17 14:35:13,700 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:35:13,984 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/duthr6v8-media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 +2025-01-17 14:35:14,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:14,700 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:15,025 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:15,206 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:16,201 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:17,207 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:17,701 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:18,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:35:18,016 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:35:18,207 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:19,208 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:20,207 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:20,458 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:21,212 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:21,703 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:22,209 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:23,209 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:23,492 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:35:23,492 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:35:23,493 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:35:23,704 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:35:23,704 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:35:23,957 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/fa8ohva5-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:35:24,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:25,217 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:25,705 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:26,153 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:26,234 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:27,230 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:27,705 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:28,227 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:29,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:30,226 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:31,218 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:31,318 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:31,707 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:32,218 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:32,656 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:35:33,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:35:33,011 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:35:33,218 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:34,218 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:35,218 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:35,708 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:36,218 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:37,133 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:37,219 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:38,219 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:39,219 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:40,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:41,219 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:41,711 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:35:42,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:42,497 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:43,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:44,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:45,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:46,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:47,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:48,011 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:35:48,011 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:35:48,167 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:48,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:49,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:50,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:51,220 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:52,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:53,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:54,035 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:54,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:55,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:56,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:57,221 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:58,239 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:58,309 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:35:58,309 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:35:58,310 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 with policy now +2025-01-17 14:35:58,310 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:35:58,310 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:35:58,311 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:35:58,554 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/u9aybhrw-media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 +2025-01-17 14:35:58,717 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:35:58,717 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 +2025-01-17 14:35:58,717 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:35:58,998 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/lwdxb3bn-media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 +2025-01-17 14:35:59,238 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:35:59,334 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:35:59,717 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:00,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:00,718 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:01,240 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:02,250 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:02,656 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:36:03,026 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:36:03,026 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:36:03,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:03,719 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:04,249 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:04,882 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:05,246 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:05,719 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:06,250 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:07,262 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:08,262 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:09,254 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:09,721 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:36:10,263 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:10,551 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:36:10,551 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:36:10,551 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:10,551 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:36:10,721 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:36:11,265 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:11,722 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:12,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:13,259 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:14,252 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:15,254 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:15,723 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:15,957 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:16,253 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:17,253 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:18,013 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:36:18,013 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:36:18,254 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:19,254 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:20,255 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:21,163 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:21,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:22,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:23,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:24,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:25,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:26,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:27,042 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:27,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:28,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:29,256 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:30,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:31,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:32,042 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:32,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:32,657 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:36:33,014 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:36:33,014 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:36:33,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:34,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:35,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:36,257 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:37,133 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:37,258 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:38,258 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:39,258 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:40,258 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:41,268 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:41,733 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:42,142 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:42,269 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:42,292 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:36:42,292 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 with policy now +2025-01-17 14:36:42,333 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:36:42,333 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:36:42,334 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:36:42,334 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:36:42,518 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/b2xifp9z-media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 +2025-01-17 14:36:42,734 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:42,734 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:36:42,734 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 +2025-01-17 14:36:42,735 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:36:43,002 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/7tx1w5el-media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 +2025-01-17 14:36:43,272 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:43,734 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:44,272 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:44,734 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:45,269 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:46,272 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:47,278 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:47,735 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:48,023 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:36:48,023 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:36:48,081 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:48,275 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:49,271 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:49,736 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:50,278 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:51,278 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:51,600 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:36:51,601 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:36:51,601 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:36:51,737 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:36:51,737 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:36:52,001 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/jgu33evp-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:36:52,291 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:53,296 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:53,738 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:54,072 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:36:54,319 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:55,292 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:56,278 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:57,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:57,739 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:36:58,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:59,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:36:59,741 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:00,062 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:00,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:01,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:02,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:02,657 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:37:03,018 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:37:03,019 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:37:03,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:04,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:05,182 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:05,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:06,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:07,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:08,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:09,279 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:10,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:11,062 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:11,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:12,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:13,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:14,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:15,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:16,063 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:16,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:17,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:18,019 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:37:18,019 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:37:18,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:19,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:20,280 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:21,143 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:21,281 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:22,281 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:23,309 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:23,750 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:23,838 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:37:23,838 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 with policy now +2025-01-17 14:37:23,848 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:37:23,849 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:37:23,849 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:37:23,849 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:37:24,069 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/fdqkbph8-media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 +2025-01-17 14:37:24,288 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:24,750 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:24,750 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:37:24,750 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 +2025-01-17 14:37:24,750 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:37:25,051 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/cz2a2lup-media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 +2025-01-17 14:37:25,301 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:25,750 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:26,306 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:26,931 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:27,303 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:28,312 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:29,313 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:29,752 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:30,315 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:31,320 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:31,753 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:32,338 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:32,658 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:37:32,659 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:33,040 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:37:33,040 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:37:33,321 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:34,337 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:35,339 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:35,650 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:37:35,650 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:37:35,651 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:37:35,754 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:37:35,754 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:37:36,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:37,345 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:37,755 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:37,769 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:38,353 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:39,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:40,339 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:41,345 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:41,756 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:42,359 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:42,833 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:43,363 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:44,364 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:45,370 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:46,353 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:47,353 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:47,759 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:37:47,924 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:48,028 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:37:48,029 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:37:48,353 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:49,353 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:50,353 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:51,353 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:52,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:53,078 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:53,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:54,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:55,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:56,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:57,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:58,078 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:37:58,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:37:59,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:00,354 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:01,355 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:02,373 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:02,658 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:38:03,049 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:38:03,049 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:38:03,162 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:03,381 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:03,629 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:38:03,629 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 with policy now +2025-01-17 14:38:03,641 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:38:03,641 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:38:03,642 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:38:03,642 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:38:03,765 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:03,765 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:38:03,765 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 +2025-01-17 14:38:03,766 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:38:03,850 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/z9tzmpyw-media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 +2025-01-17 14:38:04,116 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/njlcov0v-media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 +2025-01-17 14:38:04,390 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:04,765 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:05,382 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:05,766 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:06,383 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:07,409 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:08,408 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:09,103 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:09,446 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:09,767 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:10,409 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:11,411 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:11,768 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:12,423 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:13,426 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:14,437 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:14,459 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:14,702 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:38:14,702 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:38:14,702 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:38:14,769 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:38:14,770 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:38:15,071 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/wbsxlg8g-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:38:15,447 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:15,770 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:16,412 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:17,426 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:17,771 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:18,075 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:38:18,075 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:38:18,413 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:19,431 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:19,707 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:19,771 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:20,435 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:21,438 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:22,438 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:23,439 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:24,438 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:24,733 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:25,426 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:25,774 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:26,426 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:27,426 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:28,426 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:29,426 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:30,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:30,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:31,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:32,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:32,659 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:38:33,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:38:33,045 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:38:33,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:34,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:35,126 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:35,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:36,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:37,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:38,427 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:39,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:40,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:41,090 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:41,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:42,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:43,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:44,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:45,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:46,090 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:46,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:47,429 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:48,045 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:38:48,045 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:38:48,429 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:49,429 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:50,429 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:51,163 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:51,429 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:52,459 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:53,312 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:38:53,313 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 with policy now +2025-01-17 14:38:53,321 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:38:53,321 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:38:53,322 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:38:53,322 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:38:53,441 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:53,554 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/6myrxrf9-media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 +2025-01-17 14:38:53,785 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:53,785 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:38:53,785 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 +2025-01-17 14:38:53,785 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:38:54,080 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/k8nb6l9z-media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 +2025-01-17 14:38:54,463 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:54,785 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:55,460 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:55,786 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:56,399 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:38:56,465 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:57,495 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:57,786 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:38:58,471 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:38:59,474 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:00,476 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:01,479 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:01,788 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:02,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:02,491 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:02,659 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:39:02,759 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:39:02,759 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:39:02,765 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:39:02,788 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:39:02,789 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:39:03,061 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:39:03,061 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:39:03,493 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:03,789 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:39:03,789 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:04,493 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:05,495 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:05,790 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:06,495 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:07,497 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:07,790 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:08,061 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:08,502 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:09,519 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:10,528 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:11,524 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:12,552 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:13,123 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:13,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:13,793 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:14,513 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:15,513 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:15,794 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:16,513 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:17,513 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:18,046 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:39:18,047 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:39:18,181 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:18,513 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:19,513 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:20,513 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:21,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:22,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:23,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:24,100 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:24,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:25,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:26,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:27,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:28,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:29,100 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:29,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:30,514 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:31,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:32,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:32,660 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:39:33,047 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:39:33,047 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:39:33,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:34,197 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:34,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:35,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:36,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:37,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:38,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:39,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:40,101 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:40,515 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:41,530 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:41,619 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:39:41,619 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 with policy now +2025-01-17 14:39:41,634 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:39:41,635 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:39:41,635 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:39:41,635 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:39:41,804 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:41,804 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:39:41,804 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 +2025-01-17 14:39:41,804 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:39:41,848 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/vmjffqzc-media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 +2025-01-17 14:39:42,122 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/ajxs2v80-media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 +2025-01-17 14:39:42,528 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:42,804 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:43,530 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:43,805 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:44,532 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:45,122 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:45,530 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:46,556 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:47,556 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:47,806 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:48,079 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:39:48,079 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:39:48,538 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:49,552 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:49,807 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:50,509 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:50,554 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:51,553 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:51,808 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:39:51,821 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:39:51,821 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:39:51,822 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:39:52,555 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:52,809 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:39:52,809 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:39:53,558 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:53,809 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:54,585 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:55,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:56,056 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:39:56,568 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:57,589 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:57,811 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:39:58,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:39:59,585 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:00,583 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:01,579 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:01,812 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:40:01,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:02,579 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:02,660 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:40:03,047 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:40:03,048 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:40:03,579 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:04,579 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:05,579 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:06,580 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:07,228 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:07,580 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:08,580 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:09,580 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:10,580 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:11,580 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:12,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:13,121 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:13,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:14,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:15,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:16,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:17,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:18,047 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:40:18,048 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:40:18,184 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:18,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:19,581 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:20,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:21,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:22,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:23,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:24,122 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:24,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:25,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:26,582 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:27,621 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:28,610 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:28,944 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:40:28,944 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 with policy now +2025-01-17 14:40:28,961 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:40:28,961 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:40:28,961 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:40:28,962 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:40:29,190 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/e51sumpa-media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 +2025-01-17 14:40:29,595 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:29,823 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:40:29,824 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:40:29,824 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 +2025-01-17 14:40:29,824 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:40:30,019 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:30,123 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/ikvanlqv-media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 +2025-01-17 14:40:30,601 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:30,824 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:40:31,615 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:32,615 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:32,661 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:40:33,077 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:40:33,077 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:40:33,636 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:33,825 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:40:34,616 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:35,203 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:35,648 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:36,649 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:37,649 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:37,827 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:40:38,635 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:38,827 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:40:38,880 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:40:38,881 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:40:38,881 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:40:39,088 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/df3m9gyi-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:40:39,647 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:39,828 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:40:39,828 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:40:40,284 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:40,643 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:41,638 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:42,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:43,621 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:43,829 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:40:44,621 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:45,621 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:46,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:46,621 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:47,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:48,050 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:40:48,050 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:40:48,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:49,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:50,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:51,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:52,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:52,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:53,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:54,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:55,622 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:56,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:57,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:40:57,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:58,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:40:59,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:00,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:01,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:02,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:02,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:02,661 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:41:03,050 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:41:03,050 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:41:03,623 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:04,624 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:05,624 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:06,624 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:07,624 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:08,152 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:08,624 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:09,645 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:10,628 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:10,929 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:41:10,929 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 with policy now +2025-01-17 14:41:10,959 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:41:10,960 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:41:10,960 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:41:10,960 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:41:11,157 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/0tso4q2r-media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 +2025-01-17 14:41:11,690 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:11,841 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:11,841 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:41:11,841 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 +2025-01-17 14:41:11,841 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:41:12,146 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/z4k9ieig-media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 +2025-01-17 14:41:12,681 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:12,841 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:13,177 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:13,649 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:14,640 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:15,665 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:15,843 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:16,662 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:17,657 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:18,060 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:41:18,060 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:41:18,206 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:18,682 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:19,668 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:19,844 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:20,686 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:20,845 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:41:20,933 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:41:20,934 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:41:20,934 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:41:21,685 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:21,845 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:21,845 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:41:22,696 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:23,699 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:23,846 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:24,178 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:24,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:25,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:25,847 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:26,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:27,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:28,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:29,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:30,174 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:30,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:31,674 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:32,661 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:41:32,675 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:33,050 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:41:33,051 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:41:33,675 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:34,675 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:35,174 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:35,675 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:36,675 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:37,675 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:38,675 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:39,676 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:40,174 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:40,676 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:41,676 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:42,676 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:43,677 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:44,677 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:45,677 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:46,174 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:46,677 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:47,677 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:48,051 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:41:48,051 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:41:48,677 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:49,677 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:50,678 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:51,174 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:51,678 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:52,691 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:52,819 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:41:52,819 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 with policy now +2025-01-17 14:41:52,860 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 +2025-01-17 14:41:52,861 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:41:52,861 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:41:52,861 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:41:52,861 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:41:52,862 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:41:53,046 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/w2acgh58-media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 +2025-01-17 14:41:53,359 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/r6js5gee-media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 +2025-01-17 14:41:53,694 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:53,861 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:53,861 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:41:54,687 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:55,719 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:56,187 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:41:56,693 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:57,714 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:57,862 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:41:58,714 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:41:59,717 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:00,713 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:01,723 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:01,864 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:01,901 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:02,662 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:42:02,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:03,082 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:42:03,083 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:42:03,753 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:04,748 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:05,769 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:06,748 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:06,866 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:42:06,990 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:42:06,990 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:42:06,990 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:06,990 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:42:07,772 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:07,866 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:07,867 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:42:08,771 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:09,777 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:09,867 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:10,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:11,776 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:11,868 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:12,786 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:12,920 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:13,775 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:14,802 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:15,789 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:16,776 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:17,787 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:18,071 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:42:18,071 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:42:18,162 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:18,789 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:19,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:19,872 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:20,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:21,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:21,873 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:22,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:23,187 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:23,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:24,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:25,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:26,778 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:27,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:28,187 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:28,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:29,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:30,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:31,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:32,662 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:42:32,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:33,054 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:42:33,054 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:42:33,208 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:33,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:33,878 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:34,779 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:35,780 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:36,780 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:37,780 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:38,780 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:39,187 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:39,782 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:40,781 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:41,781 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:42,781 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:43,782 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:44,188 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:44,782 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:45,782 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:46,808 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:47,786 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:47,884 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:47,973 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:42:47,973 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 with policy now +2025-01-17 14:42:47,981 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:42:47,981 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:42:47,981 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:42:47,982 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:42:48,059 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:42:48,059 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:42:48,200 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/8tzr7loo-media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 +2025-01-17 14:42:48,842 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:48,884 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:48,885 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:42:48,885 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 +2025-01-17 14:42:48,885 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:42:49,199 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/8tgwv9hd-media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 +2025-01-17 14:42:49,210 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:49,833 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:49,885 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:50,845 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:51,848 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:52,837 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:53,854 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:53,886 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:54,222 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:54,851 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:55,831 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:55,887 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:42:56,837 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:57,836 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:57,888 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:42:58,049 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:42:58,049 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:42:58,050 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:42:58,140 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/qc3rpfq7-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:42:58,831 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:58,889 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:42:59,878 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:42:59,881 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:42:59,889 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:00,842 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:01,848 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:02,663 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:43:02,865 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:03,077 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:43:03,077 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:43:03,851 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:03,891 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:04,861 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:05,201 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:05,841 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:06,860 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:07,878 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:08,902 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:09,880 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:10,211 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:10,875 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:11,866 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:12,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:13,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:13,895 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:14,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:15,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:16,200 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:16,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:17,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:18,056 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:43:18,056 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:43:18,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:19,867 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:20,868 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:21,200 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:21,868 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:22,868 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:23,868 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:24,868 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:25,869 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:26,869 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:27,200 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:27,869 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:28,869 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:29,869 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:30,869 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:31,870 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:32,200 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:32,663 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:43:32,870 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:33,056 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:43:33,057 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:43:33,870 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:34,870 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:35,871 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:36,871 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:37,201 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:37,871 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:38,871 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:39,906 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:40,644 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:43:40,644 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 with policy now +2025-01-17 14:43:40,649 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:43:40,650 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:43:40,650 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:43:40,737 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:43:40,884 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/r1qx0bkk-media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 +2025-01-17 14:43:40,901 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:40,907 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:40,907 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:43:40,907 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 +2025-01-17 14:43:40,908 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:43:41,179 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/0qjlko87-media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 +2025-01-17 14:43:41,900 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:41,908 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:42,810 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:42,908 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:42,933 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:43,895 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:43,908 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:44,910 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:45,913 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:46,910 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:46,937 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:47,919 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:48,056 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:43:48,057 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:43:48,146 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:48,936 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:49,958 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:50,911 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:50,993 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:51,111 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:43:51,111 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:43:51,111 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:43:51,912 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:43:51,912 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:43:51,959 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:52,912 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:52,989 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:53,226 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:43:53,970 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:54,913 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:55,019 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:55,957 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:56,960 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:57,962 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:58,915 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:43:59,000 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:43:59,211 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:00,017 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:01,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:01,994 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:02,664 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:44:03,021 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:03,078 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:44:03,079 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:44:03,991 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:04,229 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:05,009 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:05,985 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:06,918 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:06,985 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:07,985 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:08,985 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:09,986 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:10,206 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:10,986 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:11,986 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:12,986 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:13,986 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:14,986 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:15,206 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:15,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:16,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:17,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:18,058 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:44:18,058 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:44:18,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:19,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:20,206 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:20,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:21,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:22,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:23,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:24,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:25,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:26,206 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:26,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:27,987 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:28,988 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:29,988 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:30,988 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:31,207 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:31,988 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:32,664 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:44:32,996 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:33,131 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:44:33,131 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:44:33,696 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:44:33,696 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 with policy now +2025-01-17 14:44:33,711 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:44:33,711 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:44:33,712 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:44:33,719 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:44:33,915 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/a81qmh8j-media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 +2025-01-17 14:44:33,930 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:44:33,930 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 +2025-01-17 14:44:33,931 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:44:33,992 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:34,207 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/43vz04pg-media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 +2025-01-17 14:44:34,931 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:35,003 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:35,931 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:36,012 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:36,834 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:36,932 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:37,020 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:38,008 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:38,932 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:39,053 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:40,036 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:40,933 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:41,015 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:41,905 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:42,022 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:42,934 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:43,049 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:43,935 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:44:44,046 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:44,176 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:44:44,177 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:44:44,177 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:44:44,935 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:44,935 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:44:45,058 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:46,043 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:46,936 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:46,993 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:47,049 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:48,061 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:48,094 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:44:48,095 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:44:49,054 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:50,050 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:50,938 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:44:51,046 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:52,051 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:52,490 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:53,053 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:54,108 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:55,136 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:56,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:57,107 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:58,037 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:44:58,104 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:44:59,096 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:00,096 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:00,942 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:01,096 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:02,096 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:02,665 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:45:02,943 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:03,070 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:45:03,070 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:45:03,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:03,218 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:04,096 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:04,944 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:05,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:06,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:07,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:08,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:08,236 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:09,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:10,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:11,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:12,097 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:13,098 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:13,237 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:14,098 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:15,098 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:16,098 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:16,949 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:17,098 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:18,071 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:45:18,071 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:45:18,113 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:19,098 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:19,236 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:20,098 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:21,099 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:22,099 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:23,099 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:24,099 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:25,099 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:25,237 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:26,119 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:26,922 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:45:26,922 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 with policy now +2025-01-17 14:45:26,922 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:45:26,923 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:45:26,923 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:26,925 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:26,954 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:26,955 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:45:26,955 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 +2025-01-17 14:45:26,955 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval +2025-01-17 14:45:27,105 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:27,155 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/0ehdgalh-media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 +2025-01-17 14:45:27,434 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/mlf2r53b-media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 +2025-01-17 14:45:28,157 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:28,955 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:29,139 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:29,956 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:30,185 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:30,260 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:31,130 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:32,165 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:32,665 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:45:32,957 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:33,089 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:45:33,089 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:45:33,176 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:34,179 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:35,145 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:35,493 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:35,958 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:45:36,179 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:36,235 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:45:36,235 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:36,236 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:36,951 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-17 14:45:36,951 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: log_artifact +2025-01-17 14:45:36,959 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:36,959 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:45:36,959 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json +2025-01-17 14:45:36,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media +2025-01-17 14:45:36,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval +2025-01-17 14:45:36,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table +2025-01-17 14:45:37,110 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:45:37,147 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:37,148 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/myl9s5ei-media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json +2025-01-17 14:45:37,428 INFO wandb-upload_2:2514330 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpph57zsi3 +2025-01-17 14:45:37,482 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-17 14:45:37,497 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:45:37,810 INFO SenderThread:2514330 [sender.py:send_request_log_artifact():1525] logged artifact run-odxnffav-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0OTEwMzMyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MTU0NTk4OQ==', 'latestArtifact': None}} +2025-01-17 14:45:37,810 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:45:37,810 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json with policy now +2025-01-17 14:45:37,810 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:45:37,810 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,811 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,811 DEBUG SenderThread:2514330 [sender.py:send():379] send: telemetry +2025-01-17 14:45:37,811 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,815 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,815 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,816 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,816 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,816 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,816 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,817 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,817 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,817 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,817 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,818 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,818 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,818 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,818 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,819 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,819 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,819 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,820 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,820 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,820 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:37,820 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:37,821 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: log_artifact +2025-01-17 14:45:37,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:45:37,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:45:37,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json +2025-01-17 14:45:37,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval +2025-01-17 14:45:38,055 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/2nmrwxif-media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json +2025-01-17 14:45:38,148 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:38,189 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/dv04cvip-media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json +2025-01-17 14:45:38,223 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/p2c0lp91-events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:45:38,234 INFO wandb-upload_2:2514330 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpl5qywjbs +2025-01-17 14:45:38,649 INFO SenderThread:2514330 [sender.py:send_request_log_artifact():1525] logged artifact run-odxnffav-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0OTEwMzM3Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MTU0NjAzMA==', 'latestArtifact': None}} +2025-01-17 14:45:38,649 DEBUG SenderThread:2514330 [sender.py:send():379] send: files +2025-01-17 14:45:38,649 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json with policy now +2025-01-17 14:45:38,649 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:45:38,650 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:38,650 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:38,904 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /tmp/tmpawxzwn98wandb/aw88p4pp-media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json +2025-01-17 14:45:38,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:38,960 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:45:39,148 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:40,148 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:40,651 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:40,961 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:41,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:42,149 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:43,025 DEBUG SenderThread:2514330 [sender.py:send():379] send: exit +2025-01-17 14:45:43,025 INFO SenderThread:2514330 [sender.py:send_exit():586] handling exit code: 0 +2025-01-17 14:45:43,026 INFO SenderThread:2514330 [sender.py:send_exit():588] handling runtime: 970 +2025-01-17 14:45:43,026 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:43,026 INFO SenderThread:2514330 [sender.py:send_exit():594] send defer +2025-01-17 14:45:43,026 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:43,026 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-17 14:45:43,026 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:43,026 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-17 14:45:43,026 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 1 +2025-01-17 14:45:43,027 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:43,027 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-17 14:45:43,027 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:43,027 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-17 14:45:43,027 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 2 +2025-01-17 14:45:43,027 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:43,027 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-17 14:45:43,027 INFO HandlerThread:2514330 [system_monitor.py:finish():203] Stopping system monitor +2025-01-17 14:45:43,027 INFO HandlerThread:2514330 [interfaces.py:finish():200] Joined cpu monitor +2025-01-17 14:45:43,027 INFO HandlerThread:2514330 [interfaces.py:finish():200] Joined disk monitor +2025-01-17 14:45:43,028 DEBUG SystemMonitor:2514330 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-17 14:45:43,028 DEBUG SystemMonitor:2514330 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-17 14:45:43,079 INFO HandlerThread:2514330 [interfaces.py:finish():200] Joined gpu monitor +2025-01-17 14:45:43,079 INFO HandlerThread:2514330 [interfaces.py:finish():200] Joined memory monitor +2025-01-17 14:45:43,079 INFO HandlerThread:2514330 [interfaces.py:finish():200] Joined network monitor +2025-01-17 14:45:43,080 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:43,080 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-17 14:45:43,080 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 3 +2025-01-17 14:45:43,080 DEBUG SenderThread:2514330 [sender.py:send():379] send: stats +2025-01-17 14:45:43,080 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:43,080 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-17 14:45:43,080 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:43,081 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-17 14:45:43,081 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 4 +2025-01-17 14:45:43,081 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:43,081 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-17 14:45:43,963 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:45:44,964 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:46,965 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/config.yaml +2025-01-17 14:45:49,242 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 14:45:49,243 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:49,243 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: keepalive +2025-01-17 14:45:49,243 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:49,244 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-17 14:45:49,244 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 5 +2025-01-17 14:45:49,244 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 14:45:49,244 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:49,244 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-17 14:45:49,244 DEBUG SenderThread:2514330 [sender.py:send():379] send: history +2025-01-17 14:45:49,245 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:45:49,246 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:49,246 DEBUG SenderThread:2514330 [sender.py:send():379] send: summary +2025-01-17 14:45:49,249 INFO SenderThread:2514330 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:45:49,250 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:49,250 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-17 14:45:49,250 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 6 +2025-01-17 14:45:49,250 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:49,250 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-17 14:45:49,250 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:49,250 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-17 14:45:49,250 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 7 +2025-01-17 14:45:49,250 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:45:49,250 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:49,250 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-17 14:45:49,251 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:49,251 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-17 14:45:49,966 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:45:50,027 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 14:45:51,043 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 8 +2025-01-17 14:45:51,044 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 14:45:51,044 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:51,044 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-17 14:45:51,044 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:51,044 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-17 14:45:51,044 INFO SenderThread:2514330 [job_builder.py:build():440] Attempting to build job artifact +2025-01-17 14:45:51,047 INFO SenderThread:2514330 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-17 14:45:51,049 INFO SenderThread:2514330 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-17 14:45:51,068 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 9 +2025-01-17 14:45:51,068 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:51,068 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-17 14:45:51,068 DEBUG SenderThread:2514330 [sender.py:send():379] send: artifact +2025-01-17 14:45:51,751 INFO wandb-upload_2:2514330 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp5pworv_0 +2025-01-17 14:45:51,752 INFO wandb-upload_3:2514330 [upload_job.py:push():85] Skipped uploading /tmp/tmpflclq4of/wandb-job.json +2025-01-17 14:45:51,872 INFO wandb-upload_0:2514330 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpxz7r8926 +2025-01-17 14:45:51,967 INFO Thread-12 :2514330 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:52,027 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 14:45:52,466 INFO SenderThread:2514330 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ0OTEwNDY1OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ0NDE0MDQxMg==', 'versionIndex': 441}}} +2025-01-17 14:45:52,466 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:52,466 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-17 14:45:52,467 INFO SenderThread:2514330 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-17 14:45:52,968 INFO SenderThread:2514330 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files +2025-01-17 14:45:52,968 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/config.yaml config.yaml +2025-01-17 14:45:52,969 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json wandb-summary.json +2025-01-17 14:45:52,970 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/requirements.txt requirements.txt +2025-01-17 14:45:52,970 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-metadata.json wandb-metadata.json +2025-01-17 14:45:52,970 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 events.out.tfevents.1737142173.leonmkim-ROG-Strix-G15CS-G15CS.2514148.0 +2025-01-17 14:45:52,970 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 14:45:52,971 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log output.log +2025-01-17 14:45:52,971 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/diff.patch diff.patch +2025-01-17 14:45:52,971 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-17 14:45:52,975 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 media/videos/eval/17_eval_17_ba7e994f26a18d93d39e.mp4 +2025-01-17 14:45:52,975 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 media/videos/eval/13_eval_13_4585080ffb0d13dad794.mp4 +2025-01-17 14:45:52,975 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 media/videos/eval/8_eval_8_c97055dba6783f8bd296.mp4 +2025-01-17 14:45:52,975 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 media/videos/eval/15_eval_15_8bb20ba2a7c41db7ced9.mp4 +2025-01-17 14:45:52,975 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 media/videos/eval/9_eval_9_1ad291b6cce26da44467.mp4 +2025-01-17 14:45:52,975 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 media/videos/eval/7_eval_7_b6f988d87c4abf5cf1bd.mp4 +2025-01-17 14:45:52,975 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 media/videos/eval/5_eval_5_b5cc7767f4cc99909240.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 media/videos/eval/10_eval_10_75a622b28aa17519a714.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 media/videos/eval/14_eval_14_9c74466fc3e808dda0ce.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 media/videos/eval/11_eval_11_e5a9c02b219899b0e912.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 media/videos/eval/0_eval_0_9913e877ba79f187dfce.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 media/videos/eval/2_eval_2_74f6d531a1bae5354927.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 media/videos/eval/12_eval_12_6c8ef51bd0e32f8f77f2.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 media/videos/eval/6_eval_6_507e5204685e3d527cd3.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 media/videos/eval/3_eval_3_90efa260cbf26eb4711e.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 media/videos/eval/1_eval_1_4d7ecef067735039e088.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 media/videos/eval/18_eval_18_d49a7e0e6acc433fa848.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 media/videos/eval/19_eval_19_b74266e52a4c9294c18d.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 media/videos/eval/4_eval_4_87ec98a2ed597fa45d1d.mp4 +2025-01-17 14:45:52,976 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 media/videos/eval/16_eval_16_2f6b9b3e7e132b67ed2e.mp4 +2025-01-17 14:45:52,977 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json media/table/eval/success_rate_ci_20_52d7fd349d2c5dd77c8a.table.json +2025-01-17 14:45:52,977 INFO SenderThread:2514330 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json media/table/eval/total_success_rate_ci_21_4a20749ae6ca4d3c4e22.table.json +2025-01-17 14:45:52,977 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 10 +2025-01-17 14:45:52,977 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 14:45:52,977 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:52,977 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-17 14:45:52,977 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:52,977 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-17 14:45:52,977 INFO SenderThread:2514330 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 14:45:53,027 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 14:45:53,028 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 14:45:53,159 INFO wandb-upload_0:2514330 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/requirements.txt +2025-01-17 14:45:53,161 INFO wandb-upload_3:2514330 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/wandb-summary.json +2025-01-17 14:45:53,161 INFO wandb-upload_1:2514330 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/config.yaml +2025-01-17 14:45:53,228 INFO wandb-upload_2:2514330 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/files/output.log +2025-01-17 14:45:53,428 INFO Thread-11 (_thread_body):2514330 [sender.py:transition_state():614] send defer: 11 +2025-01-17 14:45:53,428 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:53,429 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-17 14:45:53,429 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:53,429 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-17 14:45:53,429 INFO SenderThread:2514330 [file_pusher.py:join():175] waiting for file pusher +2025-01-17 14:45:53,434 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 12 +2025-01-17 14:45:53,434 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:53,434 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-17 14:45:53,434 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:53,434 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-17 14:45:53,435 INFO SenderThread:2514330 [file_stream.py:finish():601] file stream finish called +2025-01-17 14:45:53,699 INFO SenderThread:2514330 [file_stream.py:finish():605] file stream finish is done +2025-01-17 14:45:53,699 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 13 +2025-01-17 14:45:53,699 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:53,699 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-17 14:45:53,699 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:53,699 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-17 14:45:53,700 INFO SenderThread:2514330 [sender.py:transition_state():614] send defer: 14 +2025-01-17 14:45:53,700 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: defer +2025-01-17 14:45:53,700 INFO HandlerThread:2514330 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-17 14:45:53,700 DEBUG SenderThread:2514330 [sender.py:send():379] send: final +2025-01-17 14:45:53,700 DEBUG SenderThread:2514330 [sender.py:send():379] send: footer +2025-01-17 14:45:53,700 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: defer +2025-01-17 14:45:53,700 INFO SenderThread:2514330 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-17 14:45:53,701 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 14:45:53,701 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 14:45:53,702 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: server_info +2025-01-17 14:45:53,702 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 14:45:53,702 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: server_info +2025-01-17 14:45:53,702 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: get_summary +2025-01-17 14:45:53,704 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-17 14:45:53,705 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:45:53,766 DEBUG SenderThread:2514330 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 14:45:53,766 INFO MainThread:2514330 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-17 14:45:53,767 INFO MainThread:2514330 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-17 14:45:53,767 INFO MainThread:2514330 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-17 14:45:53,767 DEBUG HandlerThread:2514330 [handler.py:handle_request():158] handle_request: shutdown +2025-01-17 14:45:53,767 INFO HandlerThread:2514330 [handler.py:finish():882] shutting down handler +2025-01-17 14:45:54,702 INFO WriterThread:2514330 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/run-odxnffav.wandb +2025-01-17 14:45:54,766 INFO SenderThread:2514330 [sender.py:finish():1615] shutting down sender +2025-01-17 14:45:54,766 INFO SenderThread:2514330 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 14:45:54,766 INFO SenderThread:2514330 [file_pusher.py:join():175] waiting for file pusher diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/logs/debug.log b/odxnffav/wandb/run-20250117_142932-odxnffav/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..afb01a48fac18262258976b2e70104e18fb94caa --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Configure stats pid to 2514148 +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/settings +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/logs/debug.log +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/wandb/run-20250117_142932-odxnffav/logs/debug-internal.log +2025-01-17 14:29:32,134 INFO MainThread:2514148 [wandb_init.py:init():569] calling init triggers +2025-01-17 14:29:32,135 INFO MainThread:2514148 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3456_0', 'wandb_notes': '3456_0_req_3921_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav'} +2025-01-17 14:29:32,135 INFO MainThread:2514148 [wandb_init.py:init():619] starting backend +2025-01-17 14:29:32,135 INFO MainThread:2514148 [wandb_init.py:init():623] setting up manager +2025-01-17 14:29:32,139 INFO MainThread:2514148 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-17 14:29:32,140 INFO MainThread:2514148 [wandb_init.py:init():631] backend started and connected +2025-01-17 14:29:32,157 INFO MainThread:2514148 [wandb_init.py:init():720] updated telemetry +2025-01-17 14:29:32,166 INFO MainThread:2514148 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-17 14:29:32,600 INFO MainThread:2514148 [wandb_run.py:_on_init():2435] communicating current version +2025-01-17 14:29:32,638 INFO MainThread:2514148 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-17 14:29:32,638 INFO MainThread:2514148 [wandb_init.py:init():804] starting run threads in backend +2025-01-17 14:29:33,001 INFO MainThread:2514148 [wandb_run.py:_console_start():2413] atexit reg +2025-01-17 14:29:33,001 INFO MainThread:2514148 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-17 14:29:33,001 INFO MainThread:2514148 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-17 14:29:33,001 INFO MainThread:2514148 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-17 14:29:33,005 INFO MainThread:2514148 [wandb_init.py:init():847] run started, returning control to user process +2025-01-17 14:29:33,005 INFO MainThread:2514148 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/odxnffav/tb, True +2025-01-17 14:29:40,914 INFO MainThread:2514148 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-17 14:45:54,768 WARNING MsgRouterThr:2514148 [router.py:message_loop():77] message_loop has been closed diff --git a/odxnffav/wandb/run-20250117_142932-odxnffav/run-odxnffav.wandb b/odxnffav/wandb/run-20250117_142932-odxnffav/run-odxnffav.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ad12686a6fdaf41677e62e9d1040dede11f19de5 --- /dev/null +++ b/odxnffav/wandb/run-20250117_142932-odxnffav/run-odxnffav.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c77fcd04d0d8f06ee9c0196548294bd7e3e19bd8b375d9b044f4985452d35a03 +size 241442 diff --git a/roatk2bk/episode_rosbags/cam_tf_world.npy b/roatk2bk/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/roatk2bk/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/roatk2bk/episode_rosbags/depth_K.npy b/roatk2bk/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/roatk2bk/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/roatk2bk/eval_robot.log b/roatk2bk/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..15e8dba9f4f0633a7fbf8fa4a45151b35d4f3ffc --- /dev/null +++ b/roatk2bk/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-15 21:45:06,892][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-15 21:45:06,895][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/vfeb54wi/episode_rosbags/episode_0_2025-01-15-21-26-31.bag b/vfeb54wi/episode_rosbags/episode_0_2025-01-15-21-26-31.bag new file mode 100644 index 0000000000000000000000000000000000000000..4de3e622ecda1d445e22016cae2405298d2e9bf7 --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_0_2025-01-15-21-26-31.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3158aac433166cabffae1ea8d59fce094c0786458f72090c1928d1b84d505360 +size 1407565215 diff --git a/vfeb54wi/episode_rosbags/episode_11_2025-01-15-21-34-54.bag b/vfeb54wi/episode_rosbags/episode_11_2025-01-15-21-34-54.bag new file mode 100644 index 0000000000000000000000000000000000000000..471fc305f96e02bd13b43546551c41c1dc61a68f --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_11_2025-01-15-21-34-54.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f8cbee7b8840707ff6696744e933bd3f608c9e5f71cdaf14cdd1b54faca4796 +size 1411523686 diff --git a/vfeb54wi/episode_rosbags/episode_12_2025-01-15-21-35-47.bag b/vfeb54wi/episode_rosbags/episode_12_2025-01-15-21-35-47.bag new file mode 100644 index 0000000000000000000000000000000000000000..3df9a32213ca58da90ba8132c3b5c426a9c94241 --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_12_2025-01-15-21-35-47.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3624c3838b1c0648694f8da8fe9451b01947023b3c5858859f0eec55a612426 +size 1409630468 diff --git a/vfeb54wi/episode_rosbags/episode_14_2025-01-15-21-37-17.bag b/vfeb54wi/episode_rosbags/episode_14_2025-01-15-21-37-17.bag new file mode 100644 index 0000000000000000000000000000000000000000..792bc00f010be30fb673629587a8eefde6530680 --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_14_2025-01-15-21-37-17.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e82a7fc9f1a2782ffb1f6d05ad103b8e46bfd22f9e21f1f10df2b078ecf058a +size 1413966415 diff --git a/vfeb54wi/episode_rosbags/episode_15_2025-01-15-21-38-03.bag b/vfeb54wi/episode_rosbags/episode_15_2025-01-15-21-38-03.bag new file mode 100644 index 0000000000000000000000000000000000000000..1ad4f4fb294ea5d6e13c64b352fbf74a77cd719b --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_15_2025-01-15-21-38-03.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86205b776cd8db8b214da1644d69ed67e037cfdab16e287fb61efd26f3dacb62 +size 1415529761 diff --git a/vfeb54wi/episode_rosbags/episode_18_2025-01-15-21-40-48.bag b/vfeb54wi/episode_rosbags/episode_18_2025-01-15-21-40-48.bag new file mode 100644 index 0000000000000000000000000000000000000000..b4ac92eef12627252272847974ecd12c42f92bd8 --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_18_2025-01-15-21-40-48.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c8b003a29290a668e2ffeaf6b8a239ca4f1e56accbdb1afc2d37cfd068801f2 +size 1416280617 diff --git a/vfeb54wi/episode_rosbags/episode_1_2025-01-15-21-27-32.bag b/vfeb54wi/episode_rosbags/episode_1_2025-01-15-21-27-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..ec1f929e3ac74ae73316bba46456f6f5fbdf9f92 --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_1_2025-01-15-21-27-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fadb9d54309bf019e259b4b2a195b7a0d7fbaa4fd786f5120dcc99cb3cf6d5a2 +size 1412471172 diff --git a/vfeb54wi/episode_rosbags/episode_2_2025-01-15-21-28-26.bag b/vfeb54wi/episode_rosbags/episode_2_2025-01-15-21-28-26.bag new file mode 100644 index 0000000000000000000000000000000000000000..011f0bfdebae9fe3118d61c4fec9388daaa58a3f --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_2_2025-01-15-21-28-26.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ab7e70817736e900549ffe9fb5f7fb4ac8ac740b9a399f28b991027397088d4 +size 1412517673 diff --git a/vfeb54wi/episode_rosbags/episode_8_2025-01-15-21-32-40.bag b/vfeb54wi/episode_rosbags/episode_8_2025-01-15-21-32-40.bag new file mode 100644 index 0000000000000000000000000000000000000000..7932db3896720bb117892e5a1609e31794213532 --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_8_2025-01-15-21-32-40.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:591012edbb5fb79972dbc12c9b1a59bee8b832974e9e09df041c7c2d6d272f84 +size 1413784183