diff --git a/.gitattributes b/.gitattributes index a88e162b05e024413c4e1276142fcd31cfd58570..ae203203f9aba2447378b0165f0e303296eb172f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -151,3 +151,30 @@ kp7i5uxx/episode_rosbags/episode_2_2025-01-01-13-39-14.bag filter=lfs diff=lfs m 7kvn4h3l/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text 3bbvaqak/episode_rosbags/episode_2_2024-12-31-22-36-01.bag filter=lfs diff=lfs merge=lfs -text 3bbvaqak/episode_rosbags/episode_1_2024-12-31-22-34-58.bag filter=lfs diff=lfs merge=lfs -text +3bbvaqak/episode_rosbags/episode_0_2024-12-31-22-34-09.bag filter=lfs diff=lfs merge=lfs -text +uh3y27hm/episode_rosbags/episode_3_2025-01-01-13-33-05.bag filter=lfs diff=lfs merge=lfs -text +fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/3_eval_3_511ada0368931862d452.mp4 filter=lfs diff=lfs merge=lfs -text +fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 filter=lfs diff=lfs merge=lfs -text +fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 filter=lfs diff=lfs merge=lfs -text +fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 filter=lfs diff=lfs merge=lfs -text +fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 filter=lfs diff=lfs merge=lfs -text +fkfuijgf/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +fkfuijgf/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +fkfuijgf/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +fkfuijgf/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +fkfuijgf/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +aw7zqzoo/episode_rosbags/episode_0_2024-12-31-21-51-37.bag filter=lfs diff=lfs merge=lfs -text +aw7zqzoo/episode_rosbags/episode_4_2024-12-31-21-54-03.bag filter=lfs diff=lfs merge=lfs -text +aw7zqzoo/episode_rosbags/episode_3_2024-12-31-21-53-24.bag filter=lfs diff=lfs merge=lfs -text +aw7zqzoo/episode_rosbags/episode_1_2024-12-31-21-52-12.bag filter=lfs diff=lfs merge=lfs -text +aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 filter=lfs diff=lfs merge=lfs -text +aw7zqzoo/episode_rosbags/episode_2_2024-12-31-21-52-50.bag filter=lfs diff=lfs merge=lfs -text +aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 filter=lfs diff=lfs merge=lfs -text +aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 filter=lfs diff=lfs merge=lfs -text +aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 filter=lfs diff=lfs merge=lfs -text +aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 filter=lfs diff=lfs merge=lfs -text +aw7zqzoo/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +aw7zqzoo/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +aw7zqzoo/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +aw7zqzoo/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +aw7zqzoo/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/3bbvaqak/episode_rosbags/episode_0_2024-12-31-22-34-09.bag b/3bbvaqak/episode_rosbags/episode_0_2024-12-31-22-34-09.bag new file mode 100644 index 0000000000000000000000000000000000000000..6a9ad6625d6f55d9ba403627e092f4453641b952 --- /dev/null +++ b/3bbvaqak/episode_rosbags/episode_0_2024-12-31-22-34-09.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81977f462e23c739cf81a36e1f6b712bec3a559e8be3a223ae28051e1366e005 +size 2611003476 diff --git a/aw7zqzoo/.hydra/config.yaml b/aw7zqzoo/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57f0faa93e8c5310ed098ffd123244267c23fcf8 --- /dev/null +++ b/aw7zqzoo/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1745_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/aw7zqzoo/.hydra/hydra.yaml b/aw7zqzoo/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77e1917d1970064232c4b2d930efa21b7284dffd --- /dev/null +++ b/aw7zqzoo/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/aw7zqzoo/.hydra/overrides.yaml b/aw7zqzoo/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/aw7zqzoo/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/aw7zqzoo/episode_rosbags/aligned_depth_to_color_K.npy b/aw7zqzoo/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/aw7zqzoo/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/aw7zqzoo/episode_rosbags/cam_tf_world.npy b/aw7zqzoo/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/aw7zqzoo/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/aw7zqzoo/episode_rosbags/color_K.npy b/aw7zqzoo/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/aw7zqzoo/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/aw7zqzoo/episode_rosbags/depth_K.npy b/aw7zqzoo/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/aw7zqzoo/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/aw7zqzoo/episode_rosbags/episode_0_2024-12-31-21-51-37.bag b/aw7zqzoo/episode_rosbags/episode_0_2024-12-31-21-51-37.bag new file mode 100644 index 0000000000000000000000000000000000000000..d3ab3fdaecae2b59107befdab4e0320e6f91f0a2 --- /dev/null +++ b/aw7zqzoo/episode_rosbags/episode_0_2024-12-31-21-51-37.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42d9bb608aedda68c2739de3007ca29d92f7d7729a66b92c5763684d2df86091 +size 879487618 diff --git a/aw7zqzoo/episode_rosbags/episode_1_2024-12-31-21-52-12.bag b/aw7zqzoo/episode_rosbags/episode_1_2024-12-31-21-52-12.bag new file mode 100644 index 0000000000000000000000000000000000000000..8e96be006fc59671824e699cd8c5da5b19bba640 --- /dev/null +++ b/aw7zqzoo/episode_rosbags/episode_1_2024-12-31-21-52-12.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99bcd4d97686ad70b52f0491ba8683f469ed11c29fb1924c28e7d88c08723ac2 +size 945510591 diff --git a/aw7zqzoo/episode_rosbags/episode_2_2024-12-31-21-52-50.bag b/aw7zqzoo/episode_rosbags/episode_2_2024-12-31-21-52-50.bag new file mode 100644 index 0000000000000000000000000000000000000000..6448663a566beaef9a6fbce354e4d96a76fb6462 --- /dev/null +++ b/aw7zqzoo/episode_rosbags/episode_2_2024-12-31-21-52-50.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d506cedf9fab9a85a7a46bc825cdc581bcc10409eaf43b471589cbec5aa5228a +size 928514652 diff --git a/aw7zqzoo/episode_rosbags/episode_3_2024-12-31-21-53-24.bag b/aw7zqzoo/episode_rosbags/episode_3_2024-12-31-21-53-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..cbc42b101e113daff8242526dc6d02340b4dd184 --- /dev/null +++ b/aw7zqzoo/episode_rosbags/episode_3_2024-12-31-21-53-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e489ae96af2dc8661e7ae57be4d483c1884d6147568238f2691c97157ff9a892 +size 1087862198 diff --git a/aw7zqzoo/episode_rosbags/episode_4_2024-12-31-21-54-03.bag b/aw7zqzoo/episode_rosbags/episode_4_2024-12-31-21-54-03.bag new file mode 100644 index 0000000000000000000000000000000000000000..322ac9dfd4e6db2d86a81b81dbd1655eb265f25e --- /dev/null +++ b/aw7zqzoo/episode_rosbags/episode_4_2024-12-31-21-54-03.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea512ff8f69f6171f4ce63823be60ecaa5d46afaaf9f587d22bc27106cb5ecaf +size 892128511 diff --git a/aw7zqzoo/eval_robot.log b/aw7zqzoo/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..ae67e9a7dd351a5c4b5ea21da31b631b395b0ad7 --- /dev/null +++ b/aw7zqzoo/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-31 21:51:09,786][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 21:51:09,789][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/aw7zqzoo/eval_video/0_eval.mp4 b/aw7zqzoo/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c8cf430a267786d4597f0a4687d1dc92b380287 --- /dev/null +++ b/aw7zqzoo/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8294fcd60780acee658b0c6884f09caefca391b3dd175abfb5416110a12f3346 +size 583184 diff --git a/aw7zqzoo/eval_video/1_eval.mp4 b/aw7zqzoo/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bf229fc27d30851650a3997c870f37bf179d794a --- /dev/null +++ b/aw7zqzoo/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d9bf3628f96e3b0809f2e6ce385836b6392f1ac2d803546a192e8b4c98ac573 +size 597141 diff --git a/aw7zqzoo/eval_video/2_eval.mp4 b/aw7zqzoo/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7c5ec4ee77c08cc0c8d99e3715f61d8e080171dc --- /dev/null +++ b/aw7zqzoo/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a782a7003ff8f4ff060f065255aeb83d2bc77301c25cfaea78615049a270c574 +size 583314 diff --git a/aw7zqzoo/eval_video/3_eval.mp4 b/aw7zqzoo/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2cad0c547054790aa14b9d335c6e14868b537093 --- /dev/null +++ b/aw7zqzoo/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae0d1567b2384defcfc433cdecfb32afa0765bde5ace1451c776b92395063e19 +size 717452 diff --git a/aw7zqzoo/eval_video/4_eval.mp4 b/aw7zqzoo/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d66064ce4000b091bcfff341d2d6724169d41554 --- /dev/null +++ b/aw7zqzoo/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0785e49046355bcfa47b421d8d3959366a89d3906600f233fe9a7a529afc885 +size 594966 diff --git a/aw7zqzoo/tb/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 b/aw7zqzoo/tb/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 new file mode 100644 index 0000000000000000000000000000000000000000..4ec22295fde2cb1cbf816775107100256657a06d --- /dev/null +++ b/aw7zqzoo/tb/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fee80957d8bfe5ac562e2ecf9b31f61304b9dc85554194c284c13d11aa41f075 +size 6499 diff --git a/aw7zqzoo/wandb/debug-internal.log b/aw7zqzoo/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c29d4138bb9f8d4099f23e88cf4c120507df7fda --- /dev/null +++ b/aw7zqzoo/wandb/debug-internal.log @@ -0,0 +1,686 @@ +2024-12-31 21:51:15,737 INFO StreamThr :1751182 [internal.py:wandb_internal():85] W&B internal server running at pid: 1751182, started at: 2024-12-31 21:51:15.737170 +2024-12-31 21:51:15,738 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status +2024-12-31 21:51:15,755 INFO WriterThread:1751182 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/run-aw7zqzoo.wandb +2024-12-31 21:51:15,755 DEBUG SenderThread:1751182 [sender.py:send():379] send: header +2024-12-31 21:51:15,762 DEBUG SenderThread:1751182 [sender.py:send():379] send: run +2024-12-31 21:51:16,021 INFO SenderThread:1751182 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files +2024-12-31 21:51:16,021 INFO SenderThread:1751182 [sender.py:_start_run_threads():1188] run started: aw7zqzoo with start time 1735699875.740704 +2024-12-31 21:51:16,043 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 21:51:16,043 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: check_version +2024-12-31 21:51:16,089 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 21:51:16,094 DEBUG HandlerThread:1751182 [system_info.py:__init__():26] System info init +2024-12-31 21:51:16,094 DEBUG HandlerThread:1751182 [system_info.py:__init__():41] System info init done +2024-12-31 21:51:16,094 INFO HandlerThread:1751182 [system_monitor.py:start():194] Starting system monitor +2024-12-31 21:51:16,094 INFO SystemMonitor:1751182 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 21:51:16,095 INFO HandlerThread:1751182 [system_monitor.py:probe():214] Collecting system info +2024-12-31 21:51:16,095 INFO SystemMonitor:1751182 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 21:51:16,096 INFO SystemMonitor:1751182 [interfaces.py:start():188] Started disk monitoring +2024-12-31 21:51:16,096 INFO SystemMonitor:1751182 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 21:51:16,097 INFO SystemMonitor:1751182 [interfaces.py:start():188] Started memory monitoring +2024-12-31 21:51:16,097 INFO SystemMonitor:1751182 [interfaces.py:start():188] Started network monitoring +2024-12-31 21:51:16,107 DEBUG HandlerThread:1751182 [system_info.py:probe():152] Probing system +2024-12-31 21:51:16,109 DEBUG HandlerThread:1751182 [system_info.py:_probe_git():137] Probing git +2024-12-31 21:51:16,114 DEBUG HandlerThread:1751182 [system_info.py:_probe_git():145] Probing git done +2024-12-31 21:51:16,114 DEBUG HandlerThread:1751182 [system_info.py:probe():200] Probing system done +2024-12-31 21:51:16,114 DEBUG HandlerThread:1751182 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T02:51:16.107894', 'startedAt': '2025-01-01T02:51:15.729426', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3750.007875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 525.9934692382812}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 21:51:16,114 INFO HandlerThread:1751182 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 21:51:16,114 INFO HandlerThread:1751182 [system_monitor.py:probe():227] Publishing system info +2024-12-31 21:51:16,114 DEBUG HandlerThread:1751182 [system_info.py:_save_code():44] Saving code +2024-12-31 21:51:16,120 DEBUG HandlerThread:1751182 [system_info.py:_save_code():67] Saving code done +2024-12-31 21:51:16,120 DEBUG HandlerThread:1751182 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 21:51:16,185 DEBUG HandlerThread:1751182 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 21:51:16,186 INFO HandlerThread:1751182 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 21:51:16,189 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:51:16,189 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 21:51:16,189 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 21:51:16,190 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 21:51:16,432 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/wklfviyg-wandb-metadata.json +2024-12-31 21:51:16,450 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 21:51:16,450 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: python_packages +2024-12-31 21:51:16,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:51:16,451 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:51:16,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:16,453 INFO HandlerThread:1751182 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/tb" + save: true +} +control { + relay_id: "93ebf10757fb4a1fb7a5335e419c72c4" +} +_info { + stream_id: "aw7zqzoo" +} + +2024-12-31 21:51:16,453 DEBUG HandlerThread:1751182 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 21:51:16,478 INFO wandb-upload_2:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/t55opj7z-diff.patch +2024-12-31 21:51:16,480 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/wrnr7ewv-code/FISH/eval_robot.py +2024-12-31 21:51:16,519 DEBUG SenderThread:1751182 [sender.py:send():379] send: telemetry +2024-12-31 21:51:16,519 DEBUG SenderThread:1751182 [sender.py:send():379] send: tbrecord +2024-12-31 21:51:16,519 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:51:16,519 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 with policy live +2024-12-31 21:51:16,734 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/ukpnf6l2-events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:51:17,022 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:51:17,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/diff.patch +2024-12-31 21:51:17,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-metadata.json +2024-12-31 21:51:17,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:17,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/code/FISH/eval_robot.py +2024-12-31 21:51:17,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/requirements.txt +2024-12-31 21:51:17,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/code/FISH +2024-12-31 21:51:17,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/code +2024-12-31 21:51:17,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:18,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:19,022 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:19,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:20,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:21,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:21,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:22,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:23,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:23,690 DEBUG SenderThread:1751182 [sender.py:send():379] send: config +2024-12-31 21:51:24,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:25,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:25,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:26,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:26,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:27,024 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:27,457 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:28,473 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:29,024 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:29,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:30,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:31,025 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:31,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:51:31,451 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:51:31,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:31,579 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:32,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:33,026 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:33,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:34,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:35,026 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:35,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:36,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:37,027 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:37,203 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:37,457 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:38,456 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:39,027 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:39,456 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:40,456 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:41,456 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:42,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:42,456 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:43,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:44,457 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:45,457 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:46,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:51:46,452 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:51:46,493 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:47,457 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:47,606 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:48,030 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/config.yaml +2024-12-31 21:51:48,457 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:49,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:50,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:50,549 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:51:50,550 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:51:50,550 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 with policy now +2024-12-31 21:51:50,551 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:51:50,551 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:51:50,552 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:51:50,796 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/54i8b5yy-media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 +2024-12-31 21:51:51,031 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:51,031 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:51:51,031 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 +2024-12-31 21:51:51,031 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media +2024-12-31 21:51:51,031 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos +2024-12-31 21:51:51,031 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval +2024-12-31 21:51:51,319 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/shgup64e-media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 +2024-12-31 21:51:51,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:52,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:53,031 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:53,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:53,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:54,459 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:55,032 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:55,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:56,493 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:57,459 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:58,459 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:58,494 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:59,034 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:59,459 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:00,459 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:01,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:52:01,451 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:52:01,493 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:02,460 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:03,460 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:03,521 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:04,460 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:05,460 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:06,460 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:07,460 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:08,460 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:08,698 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:09,037 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:52:09,461 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:10,461 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:11,037 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:11,461 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:12,465 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:13,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:14,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:14,676 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:15,039 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:15,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:16,098 DEBUG SystemMonitor:1751182 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 21:52:16,098 DEBUG SenderThread:1751182 [sender.py:send():379] send: stats +2024-12-31 21:52:16,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:52:16,451 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:52:16,497 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:17,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:18,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:19,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:20,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:20,548 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:21,463 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:22,463 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:23,463 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:24,463 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:25,463 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:26,120 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:26,463 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:26,761 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:52:26,761 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:52:26,761 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 with policy now +2024-12-31 21:52:26,762 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:52:26,762 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:52:26,762 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:52:26,996 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/dsxfp7gp-media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 +2024-12-31 21:52:27,042 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:27,043 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:52:27,043 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 +2024-12-31 21:52:27,043 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval +2024-12-31 21:52:27,357 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/cww0bvv1-media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 +2024-12-31 21:52:27,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:28,466 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:29,043 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:29,465 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:30,470 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:31,309 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:31,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:52:31,453 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:52:31,497 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:32,471 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:33,044 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:33,466 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:34,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:35,045 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:35,468 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:36,455 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:36,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:37,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:38,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:39,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:40,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:41,455 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:41,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:42,468 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:43,468 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:44,469 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:45,468 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:46,099 DEBUG SenderThread:1751182 [sender.py:send():379] send: stats +2024-12-31 21:52:46,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:52:46,452 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:52:46,493 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:46,588 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:47,049 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:52:47,330 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/fg2bsq9r-events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:52:47,468 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:47,838 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:52:47,838 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:52:47,839 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:52:48,049 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:52:48,469 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:49,050 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:49,468 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:50,469 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:51,469 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:52,455 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:52,469 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:53,051 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:53,469 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:54,470 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:55,052 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:55,470 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:56,470 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:57,455 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:57,470 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:58,470 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:59,471 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:00,471 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:01,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:53:01,452 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:53:01,493 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:02,471 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:02,606 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:03,471 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:04,472 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:04,777 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:53:04,777 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 with policy now +2024-12-31 21:53:04,779 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:53:04,780 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:53:04,780 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:53:04,780 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:53:05,040 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/hmrzlt30-media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 +2024-12-31 21:53:05,055 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:05,056 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:53:05,056 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 +2024-12-31 21:53:05,056 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval +2024-12-31 21:53:05,326 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/hqb0v04e-media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 +2024-12-31 21:53:05,475 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:06,480 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:07,056 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:07,477 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:08,301 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:08,483 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:09,485 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:10,482 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:11,057 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:11,481 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:12,483 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:13,058 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:13,456 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:13,485 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:14,487 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:15,491 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:16,099 DEBUG SenderThread:1751182 [sender.py:send():379] send: stats +2024-12-31 21:53:16,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:53:16,453 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:53:16,497 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:17,488 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:18,492 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:18,585 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:19,491 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:20,529 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:20,661 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:53:20,661 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:53:20,661 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:53:21,061 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:53:21,061 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:53:21,495 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:22,491 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:23,061 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:23,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:23,593 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:24,491 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:25,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:26,489 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:27,063 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:27,489 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:28,489 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:28,829 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:29,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:30,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:31,065 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:31,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:53:31,452 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:53:31,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:32,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:33,065 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:33,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:34,005 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:34,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:35,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:36,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:37,491 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:38,491 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:39,457 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:39,491 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:40,493 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:41,068 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:41,404 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:53:41,404 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 with policy now +2024-12-31 21:53:41,408 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:53:41,409 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:53:41,409 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:53:41,409 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:53:41,492 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:41,645 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/vulwdj9q-media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 +2024-12-31 21:53:42,068 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:53:42,068 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 +2024-12-31 21:53:42,069 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval +2024-12-31 21:53:42,383 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/qq9vvb9d-media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 +2024-12-31 21:53:42,497 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:43,069 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:43,496 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:44,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:44,501 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:45,505 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:46,100 DEBUG SenderThread:1751182 [sender.py:send():379] send: stats +2024-12-31 21:53:46,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:53:46,462 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:53:46,505 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:47,070 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:47,501 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:48,511 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:49,071 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:49,493 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:49,503 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:50,503 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:51,505 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:52,507 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:53,506 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:54,512 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:55,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:55,510 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:56,511 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:57,507 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:58,511 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:59,512 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:00,074 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:54:00,361 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/goqxm81m-events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:54:00,516 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:00,710 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:54:00,710 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:00,710 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:00,711 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:01,075 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:01,075 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:54:01,466 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:54:01,467 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:54:01,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:02,513 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:03,075 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:03,522 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:04,513 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:05,076 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:05,513 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:06,166 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:06,513 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:07,513 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:08,513 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:09,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:10,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:11,461 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:11,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:12,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:13,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:14,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:15,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:16,100 DEBUG SenderThread:1751182 [sender.py:send():379] send: stats +2024-12-31 21:54:16,456 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:54:16,456 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:54:16,516 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:16,526 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:17,080 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:17,262 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:54:17,262 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 with policy now +2024-12-31 21:54:17,266 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:54:17,266 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:54:17,266 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:17,267 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:17,496 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/b0i7sey0-media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 +2024-12-31 21:54:17,515 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:18,080 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:54:18,081 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 +2024-12-31 21:54:18,081 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval +2024-12-31 21:54:18,364 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/e96tx633-media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 +2024-12-31 21:54:18,524 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:19,081 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:19,519 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:20,519 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:21,524 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:22,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:22,523 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:23,082 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:23,526 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:24,526 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:25,083 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:25,527 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:26,526 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:27,473 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:27,523 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:28,526 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:29,526 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:30,527 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:31,468 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:54:31,468 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:54:31,527 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:32,526 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:32,545 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:33,535 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:34,532 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:35,543 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:35,738 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:54:35,738 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:54:35,892 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:54:36,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:54:36,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:54:36,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media +2024-12-31 21:54:36,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval +2024-12-31 21:54:36,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table +2024-12-31 21:54:36,186 INFO wandb-upload_1:1751182 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvgbej2ai +2024-12-31 21:54:36,230 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:54:36,238 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:54:36,281 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/wjomz61b-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:54:36,529 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:36,844 INFO SenderThread:1751182 [sender.py:send_request_log_artifact():1525] logged artifact run-aw7zqzoo-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE0MjEwMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MDI0MA==', 'latestArtifact': None}} +2024-12-31 21:54:36,844 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:54:36,844 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,846 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,847 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:54:36,847 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2024-12-31 21:54:36,847 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:54:36,847 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,862 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,863 DEBUG SenderThread:1751182 [sender.py:send():379] send: telemetry +2024-12-31 21:54:36,871 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,872 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,872 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,885 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,885 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,909 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,909 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,926 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,927 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,952 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,953 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,964 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,964 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,973 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,973 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,985 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,986 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,997 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,997 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:37,004 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:37,005 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:37,009 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:37,010 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:54:37,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:37,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:54:37,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:54:37,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2024-12-31 21:54:37,088 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval +2024-12-31 21:54:37,137 INFO wandb-upload_2:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/v407qpq8-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:54:37,276 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/v8z0tq2s-media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2024-12-31 21:54:37,350 INFO wandb-upload_1:1751182 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0w2wyy6d +2024-12-31 21:54:37,529 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:37,726 INFO SenderThread:1751182 [sender.py:send_request_log_artifact():1525] logged artifact run-aw7zqzoo-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE0MjExMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MDI0Mw==', 'latestArtifact': None}} +2024-12-31 21:54:37,727 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:54:37,727 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json with policy now +2024-12-31 21:54:37,727 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:37,727 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:54:37,727 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:37,728 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:37,984 INFO wandb-upload_2:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/cx4dyn78-media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2024-12-31 21:54:38,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:54:38,529 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:39,088 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:39,529 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:40,529 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:40,769 DEBUG SenderThread:1751182 [sender.py:send():379] send: exit +2024-12-31 21:54:40,769 INFO SenderThread:1751182 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 21:54:40,770 INFO SenderThread:1751182 [sender.py:send_exit():588] handling runtime: 204 +2024-12-31 21:54:40,770 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:40,770 INFO SenderThread:1751182 [sender.py:send_exit():594] send defer +2024-12-31 21:54:40,770 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:40,770 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 21:54:40,770 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:40,770 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 21:54:40,770 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 1 +2024-12-31 21:54:40,770 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:40,771 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 21:54:40,771 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:40,771 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 21:54:40,771 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 2 +2024-12-31 21:54:40,771 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:40,771 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 21:54:40,771 INFO HandlerThread:1751182 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 21:54:40,772 INFO HandlerThread:1751182 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 21:54:40,772 INFO HandlerThread:1751182 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 21:54:40,772 DEBUG SystemMonitor:1751182 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 21:54:40,772 DEBUG SystemMonitor:1751182 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 21:54:40,823 INFO HandlerThread:1751182 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 21:54:40,824 INFO HandlerThread:1751182 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 21:54:40,824 INFO HandlerThread:1751182 [interfaces.py:finish():200] Joined network monitor +2024-12-31 21:54:40,825 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:40,825 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 21:54:40,825 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 3 +2024-12-31 21:54:40,825 DEBUG SenderThread:1751182 [sender.py:send():379] send: stats +2024-12-31 21:54:40,825 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:40,825 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 21:54:40,825 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:40,826 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 21:54:40,826 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 4 +2024-12-31 21:54:40,826 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:40,826 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 21:54:41,089 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:54:43,090 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:47,759 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:54:47,759 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:47,759 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 21:54:47,760 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:47,760 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 21:54:47,760 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 5 +2024-12-31 21:54:47,760 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:54:47,760 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:54:47,760 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:47,761 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:47,762 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 21:54:47,762 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:47,762 DEBUG SenderThread:1751182 [sender.py:send():379] send: summary +2024-12-31 21:54:47,766 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:47,766 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:47,767 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 21:54:47,767 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 6 +2024-12-31 21:54:47,767 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:47,767 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 21:54:47,767 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:47,767 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 21:54:47,770 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:47,771 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:54:47,895 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 7 +2024-12-31 21:54:47,896 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:54:47,896 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:47,896 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 21:54:47,896 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:47,896 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 21:54:48,091 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:54:48,092 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/config.yaml +2024-12-31 21:54:48,771 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:54:48,912 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 8 +2024-12-31 21:54:48,912 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:54:48,912 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:48,912 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 21:54:48,912 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:48,912 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 21:54:48,913 INFO SenderThread:1751182 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 21:54:48,915 INFO SenderThread:1751182 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 21:54:48,917 INFO SenderThread:1751182 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 21:54:48,930 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 9 +2024-12-31 21:54:48,930 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:48,930 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 21:54:48,931 DEBUG SenderThread:1751182 [sender.py:send():379] send: artifact +2024-12-31 21:54:49,092 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:49,472 INFO wandb-upload_2:1751182 [upload_job.py:push():85] Skipped uploading /tmp/tmpc2b6qqmm/wandb-job.json +2024-12-31 21:54:49,473 INFO wandb-upload_0:1751182 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4wyo5pt4 +2024-12-31 21:54:49,577 INFO wandb-upload_1:1751182 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprxsqpck7 +2024-12-31 21:54:49,771 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:54:50,197 INFO SenderThread:1751182 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE0MjIyOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDEzMDIwNA==', 'versionIndex': 393}}} +2024-12-31 21:54:50,197 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:50,197 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 21:54:50,197 INFO SenderThread:1751182 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 21:54:51,093 INFO SenderThread:1751182 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files +2024-12-31 21:54:51,093 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/config.yaml config.yaml +2024-12-31 21:54:51,093 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json wandb-summary.json +2024-12-31 21:54:51,094 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:54:51,098 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/requirements.txt requirements.txt +2024-12-31 21:54:51,098 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-metadata.json wandb-metadata.json +2024-12-31 21:54:51,098 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log output.log +2024-12-31 21:54:51,098 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/diff.patch diff.patch +2024-12-31 21:54:51,098 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 21:54:51,098 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 +2024-12-31 21:54:51,103 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 +2024-12-31 21:54:51,103 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 +2024-12-31 21:54:51,103 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 +2024-12-31 21:54:51,103 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 +2024-12-31 21:54:51,103 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2024-12-31 21:54:51,103 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:54:51,103 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 10 +2024-12-31 21:54:51,103 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:54:51,103 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:51,103 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 21:54:51,104 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:51,104 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 21:54:51,104 INFO SenderThread:1751182 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:54:51,308 INFO wandb-upload_2:1751182 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/config.yaml +2024-12-31 21:54:51,313 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:54:51,322 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/requirements.txt +2024-12-31 21:54:51,349 INFO wandb-upload_3:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/wg2a3phz-events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:54:51,351 INFO wandb-upload_4:1751182 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:51,551 INFO Thread-11 (_thread_body):1751182 [sender.py:transition_state():614] send defer: 11 +2024-12-31 21:54:51,552 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:51,552 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 21:54:51,552 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:51,552 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 21:54:51,552 INFO SenderThread:1751182 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 21:54:51,553 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 12 +2024-12-31 21:54:51,553 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:51,553 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 21:54:51,554 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:51,554 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 21:54:51,554 INFO SenderThread:1751182 [file_stream.py:finish():601] file stream finish called +2024-12-31 21:54:51,751 INFO SenderThread:1751182 [file_stream.py:finish():605] file stream finish is done +2024-12-31 21:54:51,751 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 13 +2024-12-31 21:54:51,751 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:51,751 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 21:54:51,751 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:51,751 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 21:54:51,751 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 14 +2024-12-31 21:54:51,751 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:51,751 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 21:54:51,752 DEBUG SenderThread:1751182 [sender.py:send():379] send: final +2024-12-31 21:54:51,752 DEBUG SenderThread:1751182 [sender.py:send():379] send: footer +2024-12-31 21:54:51,752 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:51,752 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 21:54:51,752 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:54:51,753 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 21:54:51,753 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:54:51,753 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: server_info +2024-12-31 21:54:51,754 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:54:51,755 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 21:54:51,755 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 21:54:51,756 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:51,803 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:54:51,803 INFO MainThread:1751182 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 21:54:51,804 INFO MainThread:1751182 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 21:54:51,804 INFO MainThread:1751182 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 21:54:51,804 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 21:54:51,804 INFO HandlerThread:1751182 [handler.py:finish():882] shutting down handler +2024-12-31 21:54:52,755 INFO WriterThread:1751182 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/run-aw7zqzoo.wandb +2024-12-31 21:54:52,803 INFO SenderThread:1751182 [sender.py:finish():1615] shutting down sender +2024-12-31 21:54:52,803 INFO SenderThread:1751182 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:54:52,803 INFO SenderThread:1751182 [file_pusher.py:join():175] waiting for file pusher diff --git a/aw7zqzoo/wandb/debug.log b/aw7zqzoo/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..47563c9bfcbecb86fbc885fa981a2823068abd81 --- /dev/null +++ b/aw7zqzoo/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Configure stats pid to 1750991 +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/settings +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/logs/debug.log +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/logs/debug-internal.log +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_init.py:init():569] calling init triggers +2024-12-31 21:51:15,732 INFO MainThread:1750991 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo'} +2024-12-31 21:51:15,732 INFO MainThread:1750991 [wandb_init.py:init():619] starting backend +2024-12-31 21:51:15,732 INFO MainThread:1750991 [wandb_init.py:init():623] setting up manager +2024-12-31 21:51:15,736 INFO MainThread:1750991 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 21:51:15,738 INFO MainThread:1750991 [wandb_init.py:init():631] backend started and connected +2024-12-31 21:51:15,754 INFO MainThread:1750991 [wandb_init.py:init():720] updated telemetry +2024-12-31 21:51:15,760 INFO MainThread:1750991 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 21:51:16,042 INFO MainThread:1750991 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 21:51:16,083 INFO MainThread:1750991 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 21:51:16,084 INFO MainThread:1750991 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 21:51:16,451 INFO MainThread:1750991 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 21:51:16,451 INFO MainThread:1750991 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 21:51:16,451 INFO MainThread:1750991 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 21:51:16,451 INFO MainThread:1750991 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 21:51:16,453 INFO MainThread:1750991 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 21:51:16,453 INFO MainThread:1750991 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/tb, True +2024-12-31 21:51:23,690 INFO MainThread:1750991 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-31 21:54:52,805 WARNING MsgRouterThr:1750991 [router.py:message_loop():77] message_loop has been closed diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/code/FISH/eval_robot.py b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..41791869dc3657b820d72a52a39ee14580eb36bd --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + "config.grasped_obj_name": grasped_obj_name, + "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + total_num_episodes += filtered_run.summary_metrics['episode'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/config.yaml b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92a8ee92828ac8a90cc231f3fca0570bc4daa443 --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1745_0' +wandb_notes: + desc: null + value: 1745_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735699875 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/diff.patch b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..803862adc5267428c9ad987aaa26b0cf28ac9c13 --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/diff.patch @@ -0,0 +1,54 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..57a851f 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0ada6cde6d32ac525d48de8076e15c8904b601f9 --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[0, 5, 0.0, 0, 0.5534158944976787, 0, 0.515846252438237, 0, 0.3141024113073212, 0, 0.3513819575606989]]} \ No newline at end of file diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c8cf430a267786d4597f0a4687d1dc92b380287 --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8294fcd60780acee658b0c6884f09caefca391b3dd175abfb5416110a12f3346 +size 583184 diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bf229fc27d30851650a3997c870f37bf179d794a --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d9bf3628f96e3b0809f2e6ce385836b6392f1ac2d803546a192e8b4c98ac573 +size 597141 diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7c5ec4ee77c08cc0c8d99e3715f61d8e080171dc --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a782a7003ff8f4ff060f065255aeb83d2bc77301c25cfaea78615049a270c574 +size 583314 diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2cad0c547054790aa14b9d335c6e14868b537093 --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae0d1567b2384defcfc433cdecfb32afa0765bde5ace1451c776b92395063e19 +size 717452 diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d66064ce4000b091bcfff341d2d6724169d41554 --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0785e49046355bcfa47b421d8d3959366a89d3906600f233fe9a7a529afc885 +size 594966 diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..457759da8dad0bf4c2568ea5c6aa9b5754565212 --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log @@ -0,0 +1,282 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735699883.690963]: resetting environment +[INFO] [1735699883.695143]: cleared current plan +[INFO] [1735699883.696179]: moving to home +[INFO] [1735699884.698715]: reached home +[INFO] [1735699884.701874]: reset action history +[INFO] [1735699887.416680]: environment reset +Starting episode 0 +[INFO] [1735699887.421534]: resetting environment +[INFO] [1735699887.425887]: cleared current plan +[INFO] [1735699887.426976]: moving to home +[INFO] [1735699888.429163]: reached home +[INFO] [1735699888.430456]: reset action history +[INFO] [1735699891.145617]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735699893.482547]: resetting environment +[INFO] [1735699893.487568]: cleared current plan +[INFO] [1735699893.488429]: moving to home +[INFO] [1735699894.490486]: reached home +[INFO] [1735699894.492159]: reset action history +[INFO] [1735699897.202641]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/episode_rosbags/episode_0_2024-12-31-21-51-37.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1750991.log +started roslaunch server http://158.130.50.37:36867/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1751340] +process[print_text-2]: started with pid [1751341] +process[rosbag_record-3]: started with pid [1751342] +started bagging! +preempting episode +[INFO] [1735699909.487028]: Storing episode... +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit[print_text-2] killing on exit +[INFO] [1735699910.262578]: Stored episode 1. +[INFO] [1735699910.263336]: Saving video... +[INFO] [1735699910.504160]: Video saved! +Starting episode 1 +[INFO] [1735699910.515940]: resetting environment +[INFO] [1735699910.529747]: cleared current plan +[INFO] [1735699910.533559]: moving to home +[INFO] [1735699913.740674]: reached home +[INFO] [1735699913.742701]: reset action history +[INFO] [1735699916.451752]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735699928.701982]: resetting environment +[INFO] [1735699928.707410]: cleared current plan +[INFO] [1735699928.707618]: moving to home +[INFO] [1735699929.709061]: reached home +[INFO] [1735699929.709887]: reset action history +[INFO] [1735699932.418578]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/episode_rosbags/episode_1_2024-12-31-21-52-12.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1750991.log +started roslaunch server http://158.130.50.37:34505/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1751478] +process[print_text-5]: started with pid [1751479] +process[rosbag_record-6]: started with pid [1751480] +started bagging! +preempting episode +[INFO] [1735699945.387285]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735699946.119954]: Stored episode 2. +[INFO] [1735699946.120185]: Saving video... +[INFO] [1735699946.712265]: Video saved! +Starting episode 2 +[INFO] [1735699946.746904]: resetting environment +[INFO] [1735699946.764829]: cleared current plan +[INFO] [1735699946.765069]: moving to home +[WARN] [1735699946.744998]: Plan exhausted +[INFO] [1735699950.268368]: reached home +[INFO] [1735699950.271676]: reset action history +[INFO] [1735699952.998243]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735699966.839660]: resetting environment +[INFO] [1735699966.852579]: cleared current plan +[INFO] [1735699966.853409]: moving to home +[INFO] [1735699967.860092]: reached home +[INFO] [1735699967.860720]: reset action history +[INFO] [1735699970.573463]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/episode_rosbags/episode_2_2024-12-31-21-52-50.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1750991.log +started roslaunch server http://158.130.50.37:35221/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1751621] +process[print_text-8]: started with pid [1751622] +process[rosbag_record-9]: started with pid [1751646] +started bagging! +For topic gripper_width: timestamp difference is 107410997 for nearest: 1735699972620004266 - target: 1735699972512593269 at idx 26 +preempting episode +[INFO] [1735699983.302285]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1735699984.097003]: Stored episode 3. +[INFO] [1735699984.098441]: Saving video... +[INFO] [1735699984.755033]: Video saved! +Starting episode 3 +[INFO] [1735699984.780266]: resetting environment +[INFO] [1735699984.798612]: cleared current plan +[INFO] [1735699984.799529]: moving to home +[WARN] [1735699984.585240]: Plan exhausted +[WARN] [1735699984.624818]: Plan exhausted +[WARN] [1735699984.664791]: Plan exhausted +[WARN] [1735699984.705283]: Plan exhausted +[WARN] [1735699984.744552]: Plan exhausted +[WARN] [1735699984.785970]: Plan exhausted +[INFO] [1735699988.301315]: reached home +[INFO] [1735699988.303984]: reset action history +[INFO] [1735699991.090952]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735700000.515894]: resetting environment +[INFO] [1735700000.539398]: cleared current plan +[INFO] [1735700000.541708]: moving to home +[INFO] [1735700001.549194]: reached home +[INFO] [1735700001.553244]: reset action history +[INFO] [1735700004.280436]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/episode_rosbags/episode_3_2024-12-31-21-53-24.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1750991.log +started roslaunch server http://158.130.50.37:32821/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1751783] +process[print_text-11]: started with pid [1751784] +process[rosbag_record-12]: started with pid [1751785] +started bagging! +For topic gripper_width: timestamp difference is 102504793 for nearest: 1735700008820018354 - target: 1735700008717513561 at idx 63 +For topic gripper_width: timestamp difference is 106721191 for nearest: 1735700010920012264 - target: 1735700010813291073 at idx 92 +For topic gripper_width: timestamp difference is 108815546 for nearest: 1735700011953347082 - target: 1735700011844531536 at idx 107 +preempting episode +[INFO] [1735700020.027200]: Storing episode... +[pub_text-10] killing on exit[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[INFO] [1735700020.843130]: Stored episode 4. +[INFO] [1735700020.856405]: Saving video... +[INFO] [1735700021.373962]: Video saved! +Starting episode 4 +[INFO] [1735700021.412899]: resetting environment +[INFO] [1735700021.447613]: cleared current plan +[INFO] [1735700021.449471]: moving to home +[WARN] [1735700021.344931]: Plan exhausted +[WARN] [1735700021.384544]: Plan exhausted +[WARN] [1735700021.424899]: Plan exhausted +[INFO] [1735700024.658125]: reached home +[INFO] [1735700024.662789]: reset action history +[INFO] [1735700027.451116]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735700039.979526]: resetting environment +[INFO] [1735700040.006672]: cleared current plan +[INFO] [1735700040.011953]: moving to home +[INFO] [1735700041.014865]: reached home +[INFO] [1735700041.018014]: reset action history +[INFO] [1735700043.740022]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/episode_rosbags/episode_4_2024-12-31-21-54-03.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1750991.log +started roslaunch server http://158.130.50.37:39879/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1751923] +process[print_text-14]: started with pid [1751924] +process[rosbag_record-15]: started with pid [1751925] +started bagging! +For topic gripper_width: timestamp difference is -225956469 for nearest: 1735700054753342838 - target: 1735700054979299307 at idx 159 +preempting episode +[INFO] [1735700056.114233]: Storing episode... +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1735700056.938132]: Stored episode 5. +[INFO] [1735700056.944808]: Saving video... +[INFO] [1735700057.229479]: Video saved! +[INFO] [1735700057.270886]: resetting environment +[INFO] [1735700057.293524]: cleared current plan +[INFO] [1735700057.294410]: moving to home +[WARN] [1735700057.187631]: Plan exhausted +[WARN] [1735700057.227290]: Plan exhausted +[WARN] [1735700057.265830]: Plan exhausted +[INFO] [1735700060.597373]: reached home +[INFO] [1735700060.603508]: reset action history +[INFO] [1735700063.408193]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/requirements.txt b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-metadata.json b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..fffea2847971090ddc116e93b2f77fc2bdfaef61 --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T02:51:16.107894", + "startedAt": "2025-01-01T02:51:15.729426", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3750.007875, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.998, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 525.9934692382812 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6e53318dfac896e2be76f19eb183bbe32e4a44ce --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "8294fcd60780acee658b0c6884f09caefca391b3dd175abfb5416110a12f3346", "size": 583184, "path": "media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4"}, "global_step": 242, "_timestamp": 1735700076.1479309, "_runtime": 200.40722680091858, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "5d9bf3628f96e3b0809f2e6ce385836b6392f1ac2d803546a192e8b4c98ac573", "size": 597141, "path": "media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4"}, "eval/num_success": 0.0, "episode": 5.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "a782a7003ff8f4ff060f065255aeb83d2bc77301c25cfaea78615049a270c574", "size": 583314, "path": "media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "ae0d1567b2384defcfc433cdecfb32afa0765bde5ace1451c776b92395063e19", "size": 717452, "path": "media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "a0785e49046355bcfa47b421d8d3959366a89d3906600f233fe9a7a529afc885", "size": 594966, "path": "media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://gebeni6hw7wbbwkbzilu5d9k7w52qlnht6rccbmnr1l105n78u3rc761sngs79kal1g5942obzmvmc44dt13wczwkk89upe5jms8akt0x9biw7lw8nyugknjz2cgs36x/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://qzohp9wbskwwj0injon6fqdgvfnygus6zstxx4e8x8letirdi0pb2v5hjxdm7cx34a7gcpegsvge13kmbf4cu30o95q2etwiel25azlzqhwky44f7s27rgl3izohnj19:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 0, "total_num_episodes": 5, "total_success_rate": 0.0, "total_success_rate_lb_0.01": 0, "total_success_rate_ub_0.01": 0.5534158944976787, "total_success_rate_lb_0.025": 0, "total_success_rate_ub_0.025": 0.515846252438237, "total_success_rate_lb_0.05": 0, "total_success_rate_ub_0.05": 0.3141024113073212, "total_success_rate_lb_0.1": 0, "total_success_rate_ub_0.1": 0.3513819575606989, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "3fbfe773b6a6adb9e9f84231abc953c71bfff9eaa3d69b2bf95911c7d2f3a912", "size": 434, "artifact_path": "wandb-client-artifact://2jwr994mr16zjlniqxcszpdwmxs5mmkg6aku6f18zma0n8l5rbhm4rp1unt46dumnr0d04ljbaafitmusujywx23iqmday4kqoy57mlh7e6d455o6vy90cmd7842locc/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://h9cdc9bb2dzfxfckysi8thsyz5my12ivzywyq28x785hvr4ansew0nuzvy1ofcjbub3ip9e6leo0n07evblvqtwybqyholfpxhigjpbadkc9vgnsbuo1x5hlmce6o7wd:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 204}, "eval/total_success_rate": 0.0, "eval/total_success_rate_lb_0.01": 0.0, "eval/total_success_rate_ub_0.01": 0.5534158945083618, "eval/total_success_rate_lb_0.025": 0.0, "eval/total_success_rate_ub_0.025": 0.5158462524414062, "eval/total_success_rate_lb_0.05": 0.0, "eval/total_success_rate_ub_0.05": 0.3141024112701416, "eval/total_success_rate_lb_0.1": 0.0, "eval/total_success_rate_ub_0.1": 0.35138195753097534} \ No newline at end of file diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/logs/debug-internal.log b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c29d4138bb9f8d4099f23e88cf4c120507df7fda --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/logs/debug-internal.log @@ -0,0 +1,686 @@ +2024-12-31 21:51:15,737 INFO StreamThr :1751182 [internal.py:wandb_internal():85] W&B internal server running at pid: 1751182, started at: 2024-12-31 21:51:15.737170 +2024-12-31 21:51:15,738 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status +2024-12-31 21:51:15,755 INFO WriterThread:1751182 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/run-aw7zqzoo.wandb +2024-12-31 21:51:15,755 DEBUG SenderThread:1751182 [sender.py:send():379] send: header +2024-12-31 21:51:15,762 DEBUG SenderThread:1751182 [sender.py:send():379] send: run +2024-12-31 21:51:16,021 INFO SenderThread:1751182 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files +2024-12-31 21:51:16,021 INFO SenderThread:1751182 [sender.py:_start_run_threads():1188] run started: aw7zqzoo with start time 1735699875.740704 +2024-12-31 21:51:16,043 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 21:51:16,043 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: check_version +2024-12-31 21:51:16,089 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 21:51:16,094 DEBUG HandlerThread:1751182 [system_info.py:__init__():26] System info init +2024-12-31 21:51:16,094 DEBUG HandlerThread:1751182 [system_info.py:__init__():41] System info init done +2024-12-31 21:51:16,094 INFO HandlerThread:1751182 [system_monitor.py:start():194] Starting system monitor +2024-12-31 21:51:16,094 INFO SystemMonitor:1751182 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 21:51:16,095 INFO HandlerThread:1751182 [system_monitor.py:probe():214] Collecting system info +2024-12-31 21:51:16,095 INFO SystemMonitor:1751182 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 21:51:16,096 INFO SystemMonitor:1751182 [interfaces.py:start():188] Started disk monitoring +2024-12-31 21:51:16,096 INFO SystemMonitor:1751182 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 21:51:16,097 INFO SystemMonitor:1751182 [interfaces.py:start():188] Started memory monitoring +2024-12-31 21:51:16,097 INFO SystemMonitor:1751182 [interfaces.py:start():188] Started network monitoring +2024-12-31 21:51:16,107 DEBUG HandlerThread:1751182 [system_info.py:probe():152] Probing system +2024-12-31 21:51:16,109 DEBUG HandlerThread:1751182 [system_info.py:_probe_git():137] Probing git +2024-12-31 21:51:16,114 DEBUG HandlerThread:1751182 [system_info.py:_probe_git():145] Probing git done +2024-12-31 21:51:16,114 DEBUG HandlerThread:1751182 [system_info.py:probe():200] Probing system done +2024-12-31 21:51:16,114 DEBUG HandlerThread:1751182 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T02:51:16.107894', 'startedAt': '2025-01-01T02:51:15.729426', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3750.007875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 525.9934692382812}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 21:51:16,114 INFO HandlerThread:1751182 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 21:51:16,114 INFO HandlerThread:1751182 [system_monitor.py:probe():227] Publishing system info +2024-12-31 21:51:16,114 DEBUG HandlerThread:1751182 [system_info.py:_save_code():44] Saving code +2024-12-31 21:51:16,120 DEBUG HandlerThread:1751182 [system_info.py:_save_code():67] Saving code done +2024-12-31 21:51:16,120 DEBUG HandlerThread:1751182 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 21:51:16,185 DEBUG HandlerThread:1751182 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 21:51:16,186 INFO HandlerThread:1751182 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 21:51:16,189 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:51:16,189 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 21:51:16,189 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 21:51:16,190 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 21:51:16,432 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/wklfviyg-wandb-metadata.json +2024-12-31 21:51:16,450 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 21:51:16,450 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: python_packages +2024-12-31 21:51:16,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:51:16,451 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:51:16,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:16,453 INFO HandlerThread:1751182 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/tb" + save: true +} +control { + relay_id: "93ebf10757fb4a1fb7a5335e419c72c4" +} +_info { + stream_id: "aw7zqzoo" +} + +2024-12-31 21:51:16,453 DEBUG HandlerThread:1751182 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 21:51:16,478 INFO wandb-upload_2:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/t55opj7z-diff.patch +2024-12-31 21:51:16,480 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/wrnr7ewv-code/FISH/eval_robot.py +2024-12-31 21:51:16,519 DEBUG SenderThread:1751182 [sender.py:send():379] send: telemetry +2024-12-31 21:51:16,519 DEBUG SenderThread:1751182 [sender.py:send():379] send: tbrecord +2024-12-31 21:51:16,519 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:51:16,519 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 with policy live +2024-12-31 21:51:16,734 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/ukpnf6l2-events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:51:17,022 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:51:17,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/diff.patch +2024-12-31 21:51:17,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-metadata.json +2024-12-31 21:51:17,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:17,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/code/FISH/eval_robot.py +2024-12-31 21:51:17,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/requirements.txt +2024-12-31 21:51:17,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/code/FISH +2024-12-31 21:51:17,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/code +2024-12-31 21:51:17,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:18,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:19,022 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:19,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:20,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:21,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:21,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:22,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:23,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:23,690 DEBUG SenderThread:1751182 [sender.py:send():379] send: config +2024-12-31 21:51:24,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:25,023 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:25,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:26,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:26,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:27,024 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:27,457 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:28,473 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:29,024 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:29,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:30,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:31,025 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:31,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:51:31,451 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:51:31,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:31,579 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:32,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:33,026 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:33,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:34,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:35,026 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:35,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:36,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:37,027 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:37,203 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:37,457 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:38,456 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:39,027 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:39,456 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:40,456 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:41,456 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:42,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:42,456 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:43,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:44,457 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:45,457 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:46,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:51:46,452 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:51:46,493 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:47,457 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:47,606 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:48,030 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/config.yaml +2024-12-31 21:51:48,457 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:49,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:50,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:50,549 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:51:50,550 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:51:50,550 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 with policy now +2024-12-31 21:51:50,551 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:51:50,551 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:51:50,552 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:51:50,796 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/54i8b5yy-media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 +2024-12-31 21:51:51,031 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:51,031 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:51:51,031 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 +2024-12-31 21:51:51,031 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media +2024-12-31 21:51:51,031 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos +2024-12-31 21:51:51,031 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval +2024-12-31 21:51:51,319 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/shgup64e-media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 +2024-12-31 21:51:51,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:52,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:53,031 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:53,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:53,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:54,459 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:55,032 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:55,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:56,493 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:57,459 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:58,459 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:51:58,494 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:51:59,034 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:51:59,459 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:00,459 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:01,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:52:01,451 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:52:01,493 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:02,460 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:03,460 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:03,521 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:04,460 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:05,460 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:06,460 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:07,460 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:08,460 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:08,698 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:09,037 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:52:09,461 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:10,461 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:11,037 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:11,461 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:12,465 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:13,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:14,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:14,676 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:15,039 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:15,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:16,098 DEBUG SystemMonitor:1751182 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 21:52:16,098 DEBUG SenderThread:1751182 [sender.py:send():379] send: stats +2024-12-31 21:52:16,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:52:16,451 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:52:16,497 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:17,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:18,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:19,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:20,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:20,548 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:21,463 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:22,463 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:23,463 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:24,463 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:25,463 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:26,120 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:26,463 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:26,761 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:52:26,761 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:52:26,761 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 with policy now +2024-12-31 21:52:26,762 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:52:26,762 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:52:26,762 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:52:26,996 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/dsxfp7gp-media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 +2024-12-31 21:52:27,042 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:27,043 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:52:27,043 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 +2024-12-31 21:52:27,043 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval +2024-12-31 21:52:27,357 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/cww0bvv1-media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 +2024-12-31 21:52:27,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:28,466 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:29,043 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:29,465 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:30,470 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:31,309 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:31,453 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:52:31,453 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:52:31,497 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:32,471 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:33,044 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:33,466 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:34,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:35,045 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:35,468 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:36,455 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:36,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:37,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:38,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:39,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:40,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:41,455 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:41,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:42,468 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:43,468 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:44,469 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:45,468 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:46,099 DEBUG SenderThread:1751182 [sender.py:send():379] send: stats +2024-12-31 21:52:46,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:52:46,452 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:52:46,493 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:46,588 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:47,049 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:52:47,330 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/fg2bsq9r-events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:52:47,468 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:47,838 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:52:47,838 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:52:47,839 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:52:48,049 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:52:48,469 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:49,050 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:49,468 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:50,469 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:51,469 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:52,455 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:52,469 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:53,051 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:53,469 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:54,470 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:55,052 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:52:55,470 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:56,470 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:57,455 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:52:57,470 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:58,470 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:52:59,471 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:00,471 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:01,451 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:53:01,452 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:53:01,493 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:02,471 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:02,606 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:03,471 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:04,472 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:04,777 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:53:04,777 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 with policy now +2024-12-31 21:53:04,779 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:53:04,780 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:53:04,780 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:53:04,780 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:53:05,040 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/hmrzlt30-media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 +2024-12-31 21:53:05,055 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:05,056 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:53:05,056 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 +2024-12-31 21:53:05,056 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval +2024-12-31 21:53:05,326 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/hqb0v04e-media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 +2024-12-31 21:53:05,475 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:06,480 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:07,056 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:07,477 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:08,301 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:08,483 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:09,485 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:10,482 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:11,057 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:11,481 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:12,483 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:13,058 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:13,456 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:13,485 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:14,487 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:15,491 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:16,099 DEBUG SenderThread:1751182 [sender.py:send():379] send: stats +2024-12-31 21:53:16,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:53:16,453 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:53:16,497 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:17,488 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:18,492 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:18,585 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:19,491 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:20,529 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:20,661 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:53:20,661 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:53:20,661 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:53:21,061 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:53:21,061 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:53:21,495 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:22,491 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:23,061 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:23,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:23,593 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:24,491 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:25,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:26,489 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:27,063 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:27,489 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:28,489 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:28,829 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:29,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:30,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:31,065 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:31,452 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:53:31,452 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:53:31,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:32,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:33,065 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:33,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:34,005 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:34,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:35,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:36,490 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:37,491 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:38,491 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:39,457 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:39,491 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:40,493 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:41,068 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:41,404 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:53:41,404 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 with policy now +2024-12-31 21:53:41,408 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:53:41,409 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:53:41,409 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:53:41,409 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:53:41,492 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:41,645 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/vulwdj9q-media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 +2024-12-31 21:53:42,068 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:53:42,068 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 +2024-12-31 21:53:42,069 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval +2024-12-31 21:53:42,383 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/qq9vvb9d-media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 +2024-12-31 21:53:42,497 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:43,069 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:43,496 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:44,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:44,501 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:45,505 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:46,100 DEBUG SenderThread:1751182 [sender.py:send():379] send: stats +2024-12-31 21:53:46,462 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:53:46,462 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:53:46,505 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:47,070 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:47,501 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:48,511 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:49,071 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:53:49,493 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:49,503 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:50,503 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:51,505 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:52,507 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:53,506 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:54,512 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:55,458 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:53:55,510 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:56,511 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:57,507 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:58,511 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:53:59,512 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:00,074 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:54:00,361 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/goqxm81m-events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:54:00,516 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:00,710 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:54:00,710 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:00,710 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:00,711 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:01,075 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:01,075 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:54:01,466 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:54:01,467 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:54:01,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:02,513 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:03,075 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:03,522 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:04,513 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:05,076 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:05,513 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:06,166 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:06,513 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:07,513 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:08,513 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:09,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:10,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:11,461 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:11,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:12,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:13,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:14,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:15,514 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:16,100 DEBUG SenderThread:1751182 [sender.py:send():379] send: stats +2024-12-31 21:54:16,456 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:54:16,456 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:54:16,516 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:16,526 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:17,080 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:17,262 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:54:17,262 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 with policy now +2024-12-31 21:54:17,266 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:54:17,266 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:54:17,266 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:17,267 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:17,496 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/b0i7sey0-media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 +2024-12-31 21:54:17,515 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:18,080 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:54:18,081 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 +2024-12-31 21:54:18,081 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval +2024-12-31 21:54:18,364 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/e96tx633-media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 +2024-12-31 21:54:18,524 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:19,081 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:19,519 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:20,519 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:21,524 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:22,467 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:22,523 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:23,082 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:23,526 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:24,526 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:25,083 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:25,527 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:26,526 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:27,473 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:27,523 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:28,526 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:29,526 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:30,527 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:31,468 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:54:31,468 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:54:31,527 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:32,526 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:32,545 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:33,535 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:34,532 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:35,543 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:35,738 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:54:35,738 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:54:35,892 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:54:36,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:54:36,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:54:36,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media +2024-12-31 21:54:36,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval +2024-12-31 21:54:36,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table +2024-12-31 21:54:36,186 INFO wandb-upload_1:1751182 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvgbej2ai +2024-12-31 21:54:36,230 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:54:36,238 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:54:36,281 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/wjomz61b-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:54:36,529 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:36,844 INFO SenderThread:1751182 [sender.py:send_request_log_artifact():1525] logged artifact run-aw7zqzoo-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE0MjEwMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MDI0MA==', 'latestArtifact': None}} +2024-12-31 21:54:36,844 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:54:36,844 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,846 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,847 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:54:36,847 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2024-12-31 21:54:36,847 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:54:36,847 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,862 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,863 DEBUG SenderThread:1751182 [sender.py:send():379] send: telemetry +2024-12-31 21:54:36,871 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,872 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,872 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,885 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,885 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,909 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,909 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,926 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,927 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,952 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,953 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,964 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,964 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,973 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,973 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,985 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,986 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:36,997 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:36,997 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:37,004 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:37,005 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:37,009 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:37,010 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:54:37,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:37,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:54:37,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:54:37,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2024-12-31 21:54:37,088 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval +2024-12-31 21:54:37,137 INFO wandb-upload_2:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/v407qpq8-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:54:37,276 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/v8z0tq2s-media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2024-12-31 21:54:37,350 INFO wandb-upload_1:1751182 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0w2wyy6d +2024-12-31 21:54:37,529 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:37,726 INFO SenderThread:1751182 [sender.py:send_request_log_artifact():1525] logged artifact run-aw7zqzoo-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE0MjExMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MDI0Mw==', 'latestArtifact': None}} +2024-12-31 21:54:37,727 DEBUG SenderThread:1751182 [sender.py:send():379] send: files +2024-12-31 21:54:37,727 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json with policy now +2024-12-31 21:54:37,727 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:37,727 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:54:37,727 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:37,728 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:37,984 INFO wandb-upload_2:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/cx4dyn78-media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2024-12-31 21:54:38,087 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:54:38,529 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:39,088 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:39,529 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:40,529 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:40,769 DEBUG SenderThread:1751182 [sender.py:send():379] send: exit +2024-12-31 21:54:40,769 INFO SenderThread:1751182 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 21:54:40,770 INFO SenderThread:1751182 [sender.py:send_exit():588] handling runtime: 204 +2024-12-31 21:54:40,770 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:40,770 INFO SenderThread:1751182 [sender.py:send_exit():594] send defer +2024-12-31 21:54:40,770 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:40,770 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 21:54:40,770 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:40,770 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 21:54:40,770 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 1 +2024-12-31 21:54:40,770 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:40,771 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 21:54:40,771 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:40,771 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 21:54:40,771 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 2 +2024-12-31 21:54:40,771 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:40,771 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 21:54:40,771 INFO HandlerThread:1751182 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 21:54:40,772 INFO HandlerThread:1751182 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 21:54:40,772 INFO HandlerThread:1751182 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 21:54:40,772 DEBUG SystemMonitor:1751182 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 21:54:40,772 DEBUG SystemMonitor:1751182 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 21:54:40,823 INFO HandlerThread:1751182 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 21:54:40,824 INFO HandlerThread:1751182 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 21:54:40,824 INFO HandlerThread:1751182 [interfaces.py:finish():200] Joined network monitor +2024-12-31 21:54:40,825 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:40,825 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 21:54:40,825 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 3 +2024-12-31 21:54:40,825 DEBUG SenderThread:1751182 [sender.py:send():379] send: stats +2024-12-31 21:54:40,825 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:40,825 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 21:54:40,825 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:40,826 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 21:54:40,826 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 4 +2024-12-31 21:54:40,826 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:40,826 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 21:54:41,089 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:54:43,090 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:47,759 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:54:47,759 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:47,759 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 21:54:47,760 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:47,760 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 21:54:47,760 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 5 +2024-12-31 21:54:47,760 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:54:47,760 DEBUG SenderThread:1751182 [sender.py:send():379] send: history +2024-12-31 21:54:47,760 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:54:47,761 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:47,762 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 21:54:47,762 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:47,762 DEBUG SenderThread:1751182 [sender.py:send():379] send: summary +2024-12-31 21:54:47,766 INFO SenderThread:1751182 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:54:47,766 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:47,767 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 21:54:47,767 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 6 +2024-12-31 21:54:47,767 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:47,767 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 21:54:47,767 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:47,767 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 21:54:47,770 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:54:47,771 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:54:47,895 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 7 +2024-12-31 21:54:47,896 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:54:47,896 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:47,896 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 21:54:47,896 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:47,896 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 21:54:48,091 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:54:48,092 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/config.yaml +2024-12-31 21:54:48,771 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:54:48,912 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 8 +2024-12-31 21:54:48,912 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:54:48,912 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:48,912 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 21:54:48,912 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:48,912 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 21:54:48,913 INFO SenderThread:1751182 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 21:54:48,915 INFO SenderThread:1751182 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 21:54:48,917 INFO SenderThread:1751182 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 21:54:48,930 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 9 +2024-12-31 21:54:48,930 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:48,930 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 21:54:48,931 DEBUG SenderThread:1751182 [sender.py:send():379] send: artifact +2024-12-31 21:54:49,092 INFO Thread-12 :1751182 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:49,472 INFO wandb-upload_2:1751182 [upload_job.py:push():85] Skipped uploading /tmp/tmpc2b6qqmm/wandb-job.json +2024-12-31 21:54:49,473 INFO wandb-upload_0:1751182 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4wyo5pt4 +2024-12-31 21:54:49,577 INFO wandb-upload_1:1751182 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprxsqpck7 +2024-12-31 21:54:49,771 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:54:50,197 INFO SenderThread:1751182 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE0MjIyOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDEzMDIwNA==', 'versionIndex': 393}}} +2024-12-31 21:54:50,197 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:50,197 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 21:54:50,197 INFO SenderThread:1751182 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 21:54:51,093 INFO SenderThread:1751182 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files +2024-12-31 21:54:51,093 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/config.yaml config.yaml +2024-12-31 21:54:51,093 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json wandb-summary.json +2024-12-31 21:54:51,094 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:54:51,098 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/requirements.txt requirements.txt +2024-12-31 21:54:51,098 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-metadata.json wandb-metadata.json +2024-12-31 21:54:51,098 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log output.log +2024-12-31 21:54:51,098 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/diff.patch diff.patch +2024-12-31 21:54:51,098 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 21:54:51,098 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 media/videos/eval/2_eval_2_a782a7003ff8f4ff060f.mp4 +2024-12-31 21:54:51,103 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 media/videos/eval/3_eval_3_ae0d1567b2384defcfc4.mp4 +2024-12-31 21:54:51,103 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 media/videos/eval/0_eval_0_8294fcd60780acee658b.mp4 +2024-12-31 21:54:51,103 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 media/videos/eval/1_eval_1_5d9bf3628f96e3b0809f.mp4 +2024-12-31 21:54:51,103 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 media/videos/eval/4_eval_4_a0785e49046355bcfa47.mp4 +2024-12-31 21:54:51,103 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2024-12-31 21:54:51,103 INFO SenderThread:1751182 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:54:51,103 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 10 +2024-12-31 21:54:51,103 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:54:51,103 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:51,103 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 21:54:51,104 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:51,104 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 21:54:51,104 INFO SenderThread:1751182 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:54:51,308 INFO wandb-upload_2:1751182 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/config.yaml +2024-12-31 21:54:51,313 INFO wandb-upload_0:1751182 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/wandb-summary.json +2024-12-31 21:54:51,322 INFO wandb-upload_1:1751182 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/requirements.txt +2024-12-31 21:54:51,349 INFO wandb-upload_3:1751182 [upload_job.py:push():130] Uploaded file /tmp/tmpryamckd9wandb/wg2a3phz-events.out.tfevents.1735699876.leonmkim-ROG-Strix-G15CS-G15CS.1750991.0 +2024-12-31 21:54:51,351 INFO wandb-upload_4:1751182 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/files/output.log +2024-12-31 21:54:51,551 INFO Thread-11 (_thread_body):1751182 [sender.py:transition_state():614] send defer: 11 +2024-12-31 21:54:51,552 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:51,552 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 21:54:51,552 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:51,552 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 21:54:51,552 INFO SenderThread:1751182 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 21:54:51,553 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 12 +2024-12-31 21:54:51,553 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:51,553 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 21:54:51,554 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:51,554 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 21:54:51,554 INFO SenderThread:1751182 [file_stream.py:finish():601] file stream finish called +2024-12-31 21:54:51,751 INFO SenderThread:1751182 [file_stream.py:finish():605] file stream finish is done +2024-12-31 21:54:51,751 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 13 +2024-12-31 21:54:51,751 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:51,751 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 21:54:51,751 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:51,751 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 21:54:51,751 INFO SenderThread:1751182 [sender.py:transition_state():614] send defer: 14 +2024-12-31 21:54:51,751 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:54:51,751 INFO HandlerThread:1751182 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 21:54:51,752 DEBUG SenderThread:1751182 [sender.py:send():379] send: final +2024-12-31 21:54:51,752 DEBUG SenderThread:1751182 [sender.py:send():379] send: footer +2024-12-31 21:54:51,752 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: defer +2024-12-31 21:54:51,752 INFO SenderThread:1751182 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 21:54:51,752 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:54:51,753 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 21:54:51,753 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:54:51,753 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: server_info +2024-12-31 21:54:51,754 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:54:51,755 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 21:54:51,755 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 21:54:51,756 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:54:51,803 DEBUG SenderThread:1751182 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:54:51,803 INFO MainThread:1751182 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 21:54:51,804 INFO MainThread:1751182 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 21:54:51,804 INFO MainThread:1751182 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 21:54:51,804 DEBUG HandlerThread:1751182 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 21:54:51,804 INFO HandlerThread:1751182 [handler.py:finish():882] shutting down handler +2024-12-31 21:54:52,755 INFO WriterThread:1751182 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/run-aw7zqzoo.wandb +2024-12-31 21:54:52,803 INFO SenderThread:1751182 [sender.py:finish():1615] shutting down sender +2024-12-31 21:54:52,803 INFO SenderThread:1751182 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:54:52,803 INFO SenderThread:1751182 [file_pusher.py:join():175] waiting for file pusher diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/logs/debug.log b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..47563c9bfcbecb86fbc885fa981a2823068abd81 --- /dev/null +++ b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Configure stats pid to 1750991 +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/settings +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/logs/debug.log +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/logs/debug-internal.log +2024-12-31 21:51:15,731 INFO MainThread:1750991 [wandb_init.py:init():569] calling init triggers +2024-12-31 21:51:15,732 INFO MainThread:1750991 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo'} +2024-12-31 21:51:15,732 INFO MainThread:1750991 [wandb_init.py:init():619] starting backend +2024-12-31 21:51:15,732 INFO MainThread:1750991 [wandb_init.py:init():623] setting up manager +2024-12-31 21:51:15,736 INFO MainThread:1750991 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 21:51:15,738 INFO MainThread:1750991 [wandb_init.py:init():631] backend started and connected +2024-12-31 21:51:15,754 INFO MainThread:1750991 [wandb_init.py:init():720] updated telemetry +2024-12-31 21:51:15,760 INFO MainThread:1750991 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 21:51:16,042 INFO MainThread:1750991 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 21:51:16,083 INFO MainThread:1750991 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 21:51:16,084 INFO MainThread:1750991 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 21:51:16,451 INFO MainThread:1750991 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 21:51:16,451 INFO MainThread:1750991 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 21:51:16,451 INFO MainThread:1750991 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 21:51:16,451 INFO MainThread:1750991 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 21:51:16,453 INFO MainThread:1750991 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 21:51:16,453 INFO MainThread:1750991 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/aw7zqzoo/tb, True +2024-12-31 21:51:23,690 INFO MainThread:1750991 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-31 21:54:52,805 WARNING MsgRouterThr:1750991 [router.py:message_loop():77] message_loop has been closed diff --git a/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/run-aw7zqzoo.wandb b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/run-aw7zqzoo.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4194a2fe25419e0828217d27d01654e79cf74317 Binary files /dev/null and b/aw7zqzoo/wandb/run-20241231_215115-aw7zqzoo/run-aw7zqzoo.wandb differ diff --git a/fkfuijgf/.hydra/config.yaml b/fkfuijgf/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57f0faa93e8c5310ed098ffd123244267c23fcf8 --- /dev/null +++ b/fkfuijgf/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1745_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/fkfuijgf/.hydra/hydra.yaml b/fkfuijgf/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55dca29e94be28cf989488d68eafeed4c42e5ad7 --- /dev/null +++ b/fkfuijgf/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/fkfuijgf/.hydra/overrides.yaml b/fkfuijgf/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/fkfuijgf/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/fkfuijgf/episode_rosbags/color_K.npy b/fkfuijgf/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/fkfuijgf/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/fkfuijgf/eval_video/0_eval.mp4 b/fkfuijgf/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..698b09077f762d73aca3d8dd5e044f9034f306bb --- /dev/null +++ b/fkfuijgf/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f97d134fa70e07b1df3ca6dbef126a8358e555942b7d6a536d5198e26a620b6 +size 1433247 diff --git a/fkfuijgf/eval_video/1_eval.mp4 b/fkfuijgf/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6d1af4780d17794c66b6aade5ba2f2c0069a26d5 --- /dev/null +++ b/fkfuijgf/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92c79785e0b0a9901616e727807903d0eb95a515072c2f2cfc59c47a6cbbe252 +size 1436301 diff --git a/fkfuijgf/eval_video/2_eval.mp4 b/fkfuijgf/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d6e1a19bb4b6818e08dbba8b4061a62e69ff16f7 --- /dev/null +++ b/fkfuijgf/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:942af39625588acf70b9d6f37b8e15e201d91f3e4c22855a7ac3a17c901cc2e4 +size 1462800 diff --git a/fkfuijgf/eval_video/3_eval.mp4 b/fkfuijgf/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ba63f092111fd26da446b33ec5068bcc38b51776 --- /dev/null +++ b/fkfuijgf/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:511ada0368931862d4528110e7fa8f28fcf03272a520b43d6a79c1975f2ad0be +size 1450058 diff --git a/fkfuijgf/eval_video/4_eval.mp4 b/fkfuijgf/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ba4d11d5e58876b04bd9de14bbe20491a42f72d6 --- /dev/null +++ b/fkfuijgf/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5faf9c54c3583c3620ccbaab96c58a003ca978ad55404a5d50e1b4f2764ce25 +size 1481514 diff --git a/fkfuijgf/tb/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 b/fkfuijgf/tb/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 new file mode 100644 index 0000000000000000000000000000000000000000..c8030a5fc6f14c3fd743c42fb6284092c991eab7 --- /dev/null +++ b/fkfuijgf/tb/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86c0ded846eecaee11e0f7b71f81f64afc77f98f70c22a9b4311dab7a601a0fe +size 6539 diff --git a/fkfuijgf/wandb/debug-internal.log b/fkfuijgf/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..291e9eec1607092ffc9a0b56ad60a06e89b5072e --- /dev/null +++ b/fkfuijgf/wandb/debug-internal.log @@ -0,0 +1,1055 @@ +2024-12-31 23:44:19,877 INFO StreamThr :1769274 [internal.py:wandb_internal():85] W&B internal server running at pid: 1769274, started at: 2024-12-31 23:44:19.877147 +2024-12-31 23:44:19,878 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status +2024-12-31 23:44:19,889 INFO WriterThread:1769274 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/run-fkfuijgf.wandb +2024-12-31 23:44:19,890 DEBUG SenderThread:1769274 [sender.py:send():379] send: header +2024-12-31 23:44:19,897 DEBUG SenderThread:1769274 [sender.py:send():379] send: run +2024-12-31 23:44:20,105 INFO SenderThread:1769274 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files +2024-12-31 23:44:20,105 INFO SenderThread:1769274 [sender.py:_start_run_threads():1188] run started: fkfuijgf with start time 1735706659.877195 +2024-12-31 23:44:20,120 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 23:44:20,120 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: check_version +2024-12-31 23:44:20,165 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 23:44:20,170 DEBUG HandlerThread:1769274 [system_info.py:__init__():26] System info init +2024-12-31 23:44:20,170 DEBUG HandlerThread:1769274 [system_info.py:__init__():41] System info init done +2024-12-31 23:44:20,170 INFO HandlerThread:1769274 [system_monitor.py:start():194] Starting system monitor +2024-12-31 23:44:20,170 INFO SystemMonitor:1769274 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 23:44:20,170 INFO HandlerThread:1769274 [system_monitor.py:probe():214] Collecting system info +2024-12-31 23:44:20,171 INFO SystemMonitor:1769274 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 23:44:20,172 INFO SystemMonitor:1769274 [interfaces.py:start():188] Started disk monitoring +2024-12-31 23:44:20,173 INFO SystemMonitor:1769274 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 23:44:20,175 INFO SystemMonitor:1769274 [interfaces.py:start():188] Started memory monitoring +2024-12-31 23:44:20,175 INFO SystemMonitor:1769274 [interfaces.py:start():188] Started network monitoring +2024-12-31 23:44:20,185 DEBUG HandlerThread:1769274 [system_info.py:probe():152] Probing system +2024-12-31 23:44:20,187 DEBUG HandlerThread:1769274 [system_info.py:_probe_git():137] Probing git +2024-12-31 23:44:20,192 DEBUG HandlerThread:1769274 [system_info.py:_probe_git():145] Probing git done +2024-12-31 23:44:20,192 DEBUG HandlerThread:1769274 [system_info.py:probe():200] Probing system done +2024-12-31 23:44:20,192 DEBUG HandlerThread:1769274 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T04:44:20.186033', 'startedAt': '2025-01-01T04:44:19.867291', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.4945, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.02, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 609.4676246643066}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 23:44:20,192 INFO HandlerThread:1769274 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 23:44:20,192 INFO HandlerThread:1769274 [system_monitor.py:probe():227] Publishing system info +2024-12-31 23:44:20,192 DEBUG HandlerThread:1769274 [system_info.py:_save_code():44] Saving code +2024-12-31 23:44:20,193 DEBUG HandlerThread:1769274 [system_info.py:_save_code():67] Saving code done +2024-12-31 23:44:20,193 DEBUG HandlerThread:1769274 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 23:44:20,255 DEBUG HandlerThread:1769274 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 23:44:20,255 INFO HandlerThread:1769274 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 23:44:20,259 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:44:20,259 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 23:44:20,259 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 23:44:20,259 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 23:44:20,488 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/3et9molp-wandb-metadata.json +2024-12-31 23:44:20,500 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 23:44:20,501 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: python_packages +2024-12-31 23:44:20,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:44:20,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:20,502 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:44:20,505 INFO HandlerThread:1769274 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/tb" + save: true +} +control { + relay_id: "8e3059a3c81d41b2beda0ecfc9f46d93" +} +_info { + stream_id: "fkfuijgf" +} + +2024-12-31 23:44:20,506 DEBUG HandlerThread:1769274 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 23:44:20,542 INFO wandb-upload_2:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/42d55c61-diff.patch +2024-12-31 23:44:20,543 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/m6e1crnu-code/FISH/eval_robot.py +2024-12-31 23:44:20,590 DEBUG SenderThread:1769274 [sender.py:send():379] send: telemetry +2024-12-31 23:44:20,590 DEBUG SenderThread:1769274 [sender.py:send():379] send: tbrecord +2024-12-31 23:44:20,590 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:44:20,590 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 with policy live +2024-12-31 23:44:20,786 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/0rjdmvik-events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:44:21,107 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:21,107 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/code/FISH/eval_robot.py +2024-12-31 23:44:21,107 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:44:21,107 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/diff.patch +2024-12-31 23:44:21,107 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/requirements.txt +2024-12-31 23:44:21,107 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-metadata.json +2024-12-31 23:44:21,108 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/code/FISH +2024-12-31 23:44:21,108 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/code +2024-12-31 23:44:21,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:22,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:23,106 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:23,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:24,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:24,662 DEBUG SenderThread:1769274 [sender.py:send():379] send: config +2024-12-31 23:44:25,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:44:25,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:26,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:27,107 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:27,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:28,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:29,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:30,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:44:30,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:31,108 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:31,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:32,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:33,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:34,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:35,109 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:35,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:44:35,501 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:44:35,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:35,653 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:44:36,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:37,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:38,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:39,112 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:39,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:40,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:40,661 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:44:41,506 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:42,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:43,113 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:43,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:44,506 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:45,506 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:46,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:44:46,508 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:47,506 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:48,507 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:49,507 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:50,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:44:50,501 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:44:50,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:51,507 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:51,584 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:44:52,115 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/config.yaml +2024-12-31 23:44:52,507 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:53,507 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:54,508 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:55,510 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:56,509 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:57,069 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:44:57,117 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:57,509 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:58,509 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:59,117 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:59,509 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:00,509 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:01,510 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:02,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:02,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:03,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:04,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:05,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:45:05,501 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:45:05,549 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:06,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:07,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:07,622 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:08,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:09,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:10,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:11,512 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:12,512 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:13,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:13,512 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:14,512 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:15,557 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:16,512 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:16,522 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:45:16,522 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 with policy now +2024-12-31 23:45:16,565 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:45:16,566 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:45:16,566 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:45:16,567 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:45:16,791 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/seuf4ki2-media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 +2024-12-31 23:45:17,122 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:45:17,122 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:45:17,122 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 +2024-12-31 23:45:17,122 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media +2024-12-31 23:45:17,123 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos +2024-12-31 23:45:17,123 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval +2024-12-31 23:45:17,423 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/1unwdjkl-media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 +2024-12-31 23:45:17,512 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:18,513 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:18,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:19,123 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:45:19,513 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:20,175 DEBUG SystemMonitor:1769274 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 23:45:20,176 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:45:20,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:45:20,501 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:45:20,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:21,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:22,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:23,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:23,619 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:24,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:25,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:26,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:27,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:28,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:29,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:29,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:30,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:31,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:32,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:33,127 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:45:33,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:34,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:34,577 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:35,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:45:35,502 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:45:35,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:36,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:37,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:38,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:39,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:39,603 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:40,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:41,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:42,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:43,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:44,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:45,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:45,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:46,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:47,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:48,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:49,132 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:45:49,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:50,177 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:45:50,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:45:50,501 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:45:50,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:50,563 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:51,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:52,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:53,133 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:45:53,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:54,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:55,518 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:56,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:56,518 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:57,518 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:58,518 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:59,518 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:00,518 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:01,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:02,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:02,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:03,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:04,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:05,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:46:05,501 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:46:05,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:06,138 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:46:06,413 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/erf1smsg-events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:46:06,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:07,138 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:46:07,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:07,639 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:08,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:09,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:10,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:11,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:12,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:13,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:13,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:14,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:15,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:16,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:17,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:18,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:18,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:19,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:20,177 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:46:20,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:46:20,502 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:46:20,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:21,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:22,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:23,143 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:46:23,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:23,645 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:24,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:25,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:26,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:27,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:28,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:29,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:29,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:30,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:31,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:32,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:33,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:34,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:34,523 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:35,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:46:35,502 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:46:35,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:36,523 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:37,147 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:46:37,523 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:38,523 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:39,148 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:46:39,523 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:39,639 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:40,523 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:41,149 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:46:41,523 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:42,526 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:43,528 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:44,528 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:45,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:45,528 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:46,528 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:47,528 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:48,528 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:49,528 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:50,178 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:46:50,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:46:50,502 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:46:50,549 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:50,596 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:51,529 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:52,529 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:53,529 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:54,529 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:55,529 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:55,833 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:56,530 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:57,154 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:46:57,530 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:58,530 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:59,530 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:00,530 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:01,507 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:01,531 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:02,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:03,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:04,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:05,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:47:05,502 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:47:05,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:06,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:06,655 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:07,157 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:47:07,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:08,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:09,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:10,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:11,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:12,507 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:12,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:13,533 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:14,533 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:15,540 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:15,693 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:47:15,694 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:47:15,694 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 with policy now +2024-12-31 23:47:15,695 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:47:15,695 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:47:15,695 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:47:15,968 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/zwe35cme-media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 +2024-12-31 23:47:16,160 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:47:16,160 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 +2024-12-31 23:47:16,160 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval +2024-12-31 23:47:16,448 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/ac40026d-media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 +2024-12-31 23:47:16,542 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:17,160 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:47:17,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:17,697 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:18,161 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:47:18,546 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:19,544 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:20,178 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:47:20,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:47:20,505 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:47:20,549 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:21,548 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:22,550 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:23,550 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:23,587 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:24,548 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:25,553 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:26,549 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:27,554 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:28,549 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:29,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:29,555 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:30,552 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:31,554 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:32,554 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:33,165 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:47:33,556 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:34,556 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:34,729 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:35,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:47:35,511 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:47:35,555 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:36,554 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:37,555 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:38,559 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:39,560 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:40,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:40,557 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:41,559 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:42,560 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:43,563 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:44,562 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:45,565 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:45,781 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:46,563 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:47,170 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:47:47,564 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:48,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:49,170 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:47:49,605 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:50,178 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:47:50,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:47:50,505 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:47:50,565 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:51,171 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:47:51,564 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:51,566 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:52,565 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:53,565 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:54,565 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:55,565 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:56,566 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:57,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:57,566 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:58,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:59,566 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:00,174 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:48:00,432 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/3oubdk4l-events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:48:00,566 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:01,047 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:48:01,048 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:48:01,048 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:48:01,174 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:48:01,175 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:48:01,566 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:02,569 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:03,048 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:03,175 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:48:03,570 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:04,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:05,176 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:48:05,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:48:05,505 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:48:05,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:06,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:07,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:08,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:08,582 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:09,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:10,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:11,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:12,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:13,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:14,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:14,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:15,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:16,569 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:17,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:18,569 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:19,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:19,569 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:20,179 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:48:20,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:48:20,505 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:48:20,570 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:21,570 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:22,570 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:23,570 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:24,570 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:24,620 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:25,571 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:26,571 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:27,571 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:28,572 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:29,572 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:30,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:30,572 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:31,572 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:32,572 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:33,573 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:34,572 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:35,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:48:35,505 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:48:35,572 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:35,579 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:36,573 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:37,576 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:38,405 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:48:38,405 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 with policy now +2024-12-31 23:48:38,408 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:48:38,408 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:48:38,409 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:48:38,409 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:48:38,574 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:38,680 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/frhyj3mk-media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 +2024-12-31 23:48:39,187 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:48:39,187 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:48:39,187 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 +2024-12-31 23:48:39,188 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval +2024-12-31 23:48:39,496 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/sh64xtaa-media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 +2024-12-31 23:48:39,576 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:40,187 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:48:40,577 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:41,188 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:48:41,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:41,578 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:42,577 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:43,580 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:44,579 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:45,583 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:46,584 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:47,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:47,582 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:48,581 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:49,583 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:50,179 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:48:50,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:48:50,514 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:48:50,582 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:51,583 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:52,582 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:52,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:53,584 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:54,584 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:55,584 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:56,585 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:57,587 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:57,734 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:58,590 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:59,194 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:48:59,591 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:00,587 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:01,589 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:02,594 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:03,196 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:03,518 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:03,598 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:04,599 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:05,509 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:49:05,510 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:49:05,601 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:06,601 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:07,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:07,908 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:49:07,909 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:49:07,909 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:49:08,197 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:49:08,198 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:49:08,444 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/ot7fldun-events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:49:08,603 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:09,198 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:09,469 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:09,601 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:10,605 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:11,199 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:11,601 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:12,601 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:13,199 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:13,601 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:14,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:15,408 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:15,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:16,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:17,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:18,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:19,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:20,180 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:49:20,509 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:49:20,510 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:49:20,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:20,629 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:21,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:22,603 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:23,603 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:24,603 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:25,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:26,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:26,603 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:27,603 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:28,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:29,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:30,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:31,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:31,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:32,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:33,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:34,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:35,510 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:49:35,510 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:49:35,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:36,576 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:36,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:37,606 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:38,606 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:39,606 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:40,606 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:41,607 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:42,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:42,607 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:43,607 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:44,609 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:45,210 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:45,609 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:46,620 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:47,211 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:47,331 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:49:47,332 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_511ada0368931862d452.mp4 with policy now +2024-12-31 23:49:47,335 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:49:47,335 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:49:47,335 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:49:47,357 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:49:47,605 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/r9n8aqwr-media/videos/eval/3_eval_3_511ada0368931862d452.mp4 +2024-12-31 23:49:47,623 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:48,211 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:48,212 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:49:48,212 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/3_eval_3_511ada0368931862d452.mp4 +2024-12-31 23:49:48,212 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval +2024-12-31 23:49:48,376 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:48,568 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/p0kbzqpj-media/videos/eval/3_eval_3_511ada0368931862d452.mp4 +2024-12-31 23:49:48,611 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:49,212 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:49,612 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:50,180 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:49:50,212 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:50,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:49:50,520 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:49:50,620 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:51,615 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:52,613 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:53,600 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:53,614 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:54,614 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:55,623 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:56,623 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:57,624 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:58,622 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:59,526 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:59,629 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:00,622 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:01,628 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:02,628 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:03,216 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:50:03,624 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:04,628 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:05,429 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:05,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:50:05,516 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:50:05,624 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:06,628 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:07,625 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:08,629 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:09,634 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:10,530 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:10,642 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:11,640 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:12,640 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:13,219 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:50:13,650 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:14,641 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:15,646 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:16,530 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:16,644 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:17,221 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:50:17,648 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:18,644 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:19,646 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:20,181 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:50:20,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:50:20,522 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:50:20,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:21,651 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:21,672 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:22,654 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:23,655 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:24,656 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:25,658 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:26,661 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:27,533 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:27,658 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:28,224 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:50:28,663 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:29,003 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:50:29,003 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:50:29,003 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:50:29,225 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:50:29,225 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:50:29,225 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:50:29,476 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/iq5zua9g-events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:50:29,661 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:30,666 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:31,666 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:32,666 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:33,533 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:33,664 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:34,666 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:35,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:50:35,520 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:50:35,667 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:36,665 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:37,667 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:38,607 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:38,668 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:39,671 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:40,669 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:41,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:42,676 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:43,675 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:44,467 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:44,668 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:45,230 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:50:45,672 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:46,676 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:47,230 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:50:47,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:48,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:49,231 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:50:49,510 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:49,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:50,181 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:50:50,513 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:50:50,514 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:50:50,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:51,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:52,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:53,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:54,653 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:54,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:55,671 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:56,671 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:57,671 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:58,671 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:59,671 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:00,485 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:00,674 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:01,235 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:01,672 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:02,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:03,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:04,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:05,513 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:51:05,514 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:51:05,618 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:05,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:06,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:07,238 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:07,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:08,674 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:09,239 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:09,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:10,674 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:11,531 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:11,674 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:12,674 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:13,240 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:13,675 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:14,675 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:15,241 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:15,675 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:16,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:16,675 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:17,677 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:18,676 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:19,242 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:19,676 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:20,182 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:51:20,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:51:20,514 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:51:20,676 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:21,665 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:21,676 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:22,692 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:23,244 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:23,491 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:51:23,491 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 with policy now +2024-12-31 23:51:23,493 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:51:23,494 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:51:23,494 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:51:23,494 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:51:23,682 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:23,780 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/tfgiibxx-media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 +2024-12-31 23:51:24,244 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:24,244 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:51:24,245 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 +2024-12-31 23:51:24,245 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval +2024-12-31 23:51:24,554 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/3cg14ctk-media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 +2024-12-31 23:51:24,691 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:25,245 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:25,688 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:26,245 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:26,691 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:27,537 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:27,698 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:28,706 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:29,702 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:30,702 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:31,710 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:32,548 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:32,710 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:33,714 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:34,717 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:35,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:51:35,520 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:51:35,710 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:36,721 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:37,580 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:37,714 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:38,717 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:39,249 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:39,716 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:40,723 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:41,727 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:42,729 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:43,541 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:43,732 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:44,734 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:45,732 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:46,727 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:47,728 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:48,547 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:48,726 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:49,729 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:50,182 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:51:50,526 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:51:50,526 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:51:50,738 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:51,732 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:52,731 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:53,612 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:53,731 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:54,734 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:55,255 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:55,735 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:56,740 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:57,256 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:57,732 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:58,737 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:59,256 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:59,477 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:59,732 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:00,731 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:01,110 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:52:01,110 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:01,110 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:01,257 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:52:01,257 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:52:01,257 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:52:01,300 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:52:01,300 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:52:01,512 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:52:01,520 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/birw3f9s-events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:52:01,731 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:01,767 INFO wandb-upload_0:1769274 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp8i279p53 +2024-12-31 23:52:01,890 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:52:01,900 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:52:02,128 INFO SenderThread:1769274 [sender.py:send_request_log_artifact():1525] logged artifact run-fkfuijgf-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIxNDAwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3OTYzMQ==', 'latestArtifact': None}} +2024-12-31 23:52:02,128 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:52:02,128 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2024-12-31 23:52:02,129 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:52:02,129 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,129 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,129 DEBUG SenderThread:1769274 [sender.py:send():379] send: telemetry +2024-12-31 23:52:02,129 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,130 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,130 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,130 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,130 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,132 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,132 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,133 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,133 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,133 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,133 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,134 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,134 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,134 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,134 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,135 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,135 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,135 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,135 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,136 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,136 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,136 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,136 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:52:02,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:52:02,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:52:02,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json +2024-12-31 23:52:02,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:52:02,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media +2024-12-31 23:52:02,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval +2024-12-31 23:52:02,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table +2024-12-31 23:52:02,328 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/y2b70qk6-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:52:02,496 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/wy349f5u-media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json +2024-12-31 23:52:02,548 INFO wandb-upload_2:1769274 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp58yspos_ +2024-12-31 23:52:02,607 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/gue7j1tz-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:52:02,731 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:03,040 INFO SenderThread:1769274 [sender.py:send_request_log_artifact():1525] logged artifact run-fkfuijgf-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIxNDAxMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3OTYzMg==', 'latestArtifact': None}} +2024-12-31 23:52:03,040 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:52:03,040 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json with policy now +2024-12-31 23:52:03,040 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:52:03,040 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:03,041 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:03,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:52:03,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:52:03,294 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/5nmobrxv-media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json +2024-12-31 23:52:03,645 DEBUG SenderThread:1769274 [sender.py:send():379] send: exit +2024-12-31 23:52:03,645 INFO SenderThread:1769274 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 23:52:03,646 INFO SenderThread:1769274 [sender.py:send_exit():588] handling runtime: 463 +2024-12-31 23:52:03,646 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:03,646 INFO SenderThread:1769274 [sender.py:send_exit():594] send defer +2024-12-31 23:52:03,646 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:03,646 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 23:52:03,646 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:03,646 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 23:52:03,646 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 1 +2024-12-31 23:52:03,647 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:03,647 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 23:52:03,647 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:03,647 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 23:52:03,647 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 2 +2024-12-31 23:52:03,647 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:03,647 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 23:52:03,647 INFO HandlerThread:1769274 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 23:52:03,647 DEBUG SystemMonitor:1769274 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 23:52:03,647 DEBUG SystemMonitor:1769274 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 23:52:03,648 INFO HandlerThread:1769274 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 23:52:03,648 INFO HandlerThread:1769274 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 23:52:03,699 INFO HandlerThread:1769274 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 23:52:03,699 INFO HandlerThread:1769274 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 23:52:03,699 INFO HandlerThread:1769274 [interfaces.py:finish():200] Joined network monitor +2024-12-31 23:52:03,700 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:03,700 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 23:52:03,700 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 3 +2024-12-31 23:52:03,700 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:52:03,700 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:03,700 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 23:52:03,700 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:03,700 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 23:52:03,700 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 4 +2024-12-31 23:52:03,701 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:03,701 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 23:52:04,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:52:05,259 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:52:10,115 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:52:10,116 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:10,116 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 23:52:10,116 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:10,117 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:10,117 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 23:52:10,118 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 5 +2024-12-31 23:52:10,118 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:52:10,118 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:52:10,118 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:10,118 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:10,118 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 23:52:10,119 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:10,119 DEBUG SenderThread:1769274 [sender.py:send():379] send: summary +2024-12-31 23:52:10,122 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:10,123 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:10,123 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 23:52:10,123 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 6 +2024-12-31 23:52:10,123 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:10,123 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 23:52:10,123 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:10,123 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 23:52:10,126 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:10,251 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 7 +2024-12-31 23:52:10,251 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:10,251 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 23:52:10,251 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:10,251 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 23:52:10,261 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:52:10,261 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/config.yaml +2024-12-31 23:52:10,647 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:52:11,972 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 8 +2024-12-31 23:52:11,972 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:52:11,972 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:11,973 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 23:52:11,973 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:11,973 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 23:52:11,973 INFO SenderThread:1769274 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 23:52:11,975 INFO SenderThread:1769274 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 23:52:11,977 INFO SenderThread:1769274 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 23:52:11,990 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 9 +2024-12-31 23:52:11,991 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:11,991 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 23:52:11,991 DEBUG SenderThread:1769274 [sender.py:send():379] send: artifact +2024-12-31 23:52:12,157 INFO SenderThread:1769274 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIwNzU2Mg==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDIwNzU2Mg==', 'versionIndex': 401}}} +2024-12-31 23:52:12,157 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:12,157 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 23:52:12,157 INFO SenderThread:1769274 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 23:52:12,262 INFO SenderThread:1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:52:12,262 INFO SenderThread:1769274 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files +2024-12-31 23:52:12,262 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/config.yaml config.yaml +2024-12-31 23:52:12,263 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:52:12,263 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json wandb-summary.json +2024-12-31 23:52:12,264 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/requirements.txt requirements.txt +2024-12-31 23:52:12,265 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-metadata.json wandb-metadata.json +2024-12-31 23:52:12,265 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log output.log +2024-12-31 23:52:12,265 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/diff.patch diff.patch +2024-12-31 23:52:12,265 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 23:52:12,269 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 +2024-12-31 23:52:12,270 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 +2024-12-31 23:52:12,270 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 +2024-12-31 23:52:12,271 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 +2024-12-31 23:52:12,271 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/3_eval_3_511ada0368931862d452.mp4 media/videos/eval/3_eval_3_511ada0368931862d452.mp4 +2024-12-31 23:52:12,271 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:52:12,271 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json +2024-12-31 23:52:12,271 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 10 +2024-12-31 23:52:12,271 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:12,271 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 23:52:12,272 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:12,272 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 23:52:12,272 INFO SenderThread:1769274 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:52:12,461 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/config.yaml +2024-12-31 23:52:12,463 INFO wandb-upload_2:1769274 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:52:12,499 INFO wandb-upload_4:1769274 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:52:12,505 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/mdgf2ixm-events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:52:12,509 INFO wandb-upload_3:1769274 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/requirements.txt +2024-12-31 23:52:12,647 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:52:12,648 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:52:12,709 INFO Thread-11 (_thread_body):1769274 [sender.py:transition_state():614] send defer: 11 +2024-12-31 23:52:12,709 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:12,709 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 23:52:12,709 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:12,709 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 23:52:12,710 INFO SenderThread:1769274 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 23:52:12,711 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 12 +2024-12-31 23:52:12,711 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:12,711 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 23:52:12,711 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:12,711 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 23:52:12,711 INFO SenderThread:1769274 [file_stream.py:finish():601] file stream finish called +2024-12-31 23:52:12,896 INFO SenderThread:1769274 [file_stream.py:finish():605] file stream finish is done +2024-12-31 23:52:12,896 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 13 +2024-12-31 23:52:12,896 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:12,896 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 23:52:12,896 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:12,896 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 23:52:12,897 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 14 +2024-12-31 23:52:12,897 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:12,897 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 23:52:12,897 DEBUG SenderThread:1769274 [sender.py:send():379] send: final +2024-12-31 23:52:12,897 DEBUG SenderThread:1769274 [sender.py:send():379] send: footer +2024-12-31 23:52:12,897 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:12,897 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 23:52:12,898 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:52:12,898 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 23:52:12,898 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:52:12,898 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:52:12,898 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 23:52:12,898 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: server_info +2024-12-31 23:52:12,899 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 23:52:12,901 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:12,950 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:52:12,950 INFO MainThread:1769274 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 23:52:12,950 INFO MainThread:1769274 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 23:52:12,951 INFO MainThread:1769274 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 23:52:12,951 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 23:52:12,951 INFO HandlerThread:1769274 [handler.py:finish():882] shutting down handler +2024-12-31 23:52:13,898 INFO WriterThread:1769274 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/run-fkfuijgf.wandb +2024-12-31 23:52:13,950 INFO SenderThread:1769274 [sender.py:finish():1615] shutting down sender +2024-12-31 23:52:13,950 INFO SenderThread:1769274 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:52:13,950 INFO SenderThread:1769274 [file_pusher.py:join():175] waiting for file pusher diff --git a/fkfuijgf/wandb/debug.log b/fkfuijgf/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a6e1076d8261a135e7f9c06f57bf32d0b05bc914 --- /dev/null +++ b/fkfuijgf/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-31 23:44:19,869 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 23:44:19,869 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Configure stats pid to 1769082 +2024-12-31 23:44:19,869 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 23:44:19,869 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/settings +2024-12-31 23:44:19,870 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 23:44:19,870 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 23:44:19,870 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 23:44:19,870 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 23:44:19,871 INFO MainThread:1769082 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/logs/debug.log +2024-12-31 23:44:19,871 INFO MainThread:1769082 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/logs/debug-internal.log +2024-12-31 23:44:19,871 INFO MainThread:1769082 [wandb_init.py:init():569] calling init triggers +2024-12-31 23:44:19,872 INFO MainThread:1769082 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf'} +2024-12-31 23:44:19,872 INFO MainThread:1769082 [wandb_init.py:init():619] starting backend +2024-12-31 23:44:19,872 INFO MainThread:1769082 [wandb_init.py:init():623] setting up manager +2024-12-31 23:44:19,876 INFO MainThread:1769082 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 23:44:19,877 INFO MainThread:1769082 [wandb_init.py:init():631] backend started and connected +2024-12-31 23:44:19,889 INFO MainThread:1769082 [wandb_init.py:init():720] updated telemetry +2024-12-31 23:44:19,895 INFO MainThread:1769082 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 23:44:20,120 INFO MainThread:1769082 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 23:44:20,158 INFO MainThread:1769082 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 23:44:20,158 INFO MainThread:1769082 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 23:44:20,501 INFO MainThread:1769082 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 23:44:20,501 INFO MainThread:1769082 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 23:44:20,501 INFO MainThread:1769082 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 23:44:20,501 INFO MainThread:1769082 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 23:44:20,502 INFO MainThread:1769082 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 23:44:20,502 INFO MainThread:1769082 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/tb, True +2024-12-31 23:44:24,662 INFO MainThread:1769082 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2024-12-31 23:52:13,952 WARNING MsgRouterThr:1769082 [router.py:message_loop():77] message_loop has been closed diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/code/FISH/eval_robot.py b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/config.yaml b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59b76e41eb57dc76b6aa29b2efb44f329b96c2e0 --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1745_0' +wandb_notes: + desc: null + value: 1745_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735706659 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/diff.patch b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..d0fe8478ad7c175b306ffe80f6ea9aaa7ecdf9ce --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/diff.patch @@ -0,0 +1,102 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..57a851f 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..3db3932 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 15.0 diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json new file mode 100644 index 0000000000000000000000000000000000000000..7d31a2b832c10253dc7954e26c2c95338dee9b16 --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[10, 25, 0.4, 0.19247555738877542, 0.6353677510944474, 0.23085720842499102, 0.6117631792798629, 0.25551603739382933, 0.5599231943368275, 0.27121893693611987, 0.5436965897591937]]} \ No newline at end of file diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..698b09077f762d73aca3d8dd5e044f9034f306bb --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f97d134fa70e07b1df3ca6dbef126a8358e555942b7d6a536d5198e26a620b6 +size 1433247 diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6d1af4780d17794c66b6aade5ba2f2c0069a26d5 --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92c79785e0b0a9901616e727807903d0eb95a515072c2f2cfc59c47a6cbbe252 +size 1436301 diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d6e1a19bb4b6818e08dbba8b4061a62e69ff16f7 --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:942af39625588acf70b9d6f37b8e15e201d91f3e4c22855a7ac3a17c901cc2e4 +size 1462800 diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/3_eval_3_511ada0368931862d452.mp4 b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/3_eval_3_511ada0368931862d452.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ba63f092111fd26da446b33ec5068bcc38b51776 --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/3_eval_3_511ada0368931862d452.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:511ada0368931862d4528110e7fa8f28fcf03272a520b43d6a79c1975f2ad0be +size 1450058 diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ba4d11d5e58876b04bd9de14bbe20491a42f72d6 --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5faf9c54c3583c3620ccbaab96c58a003ca978ad55404a5d50e1b4f2764ce25 +size 1481514 diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..2ba3938f52ec9cdadb1a20589d1466e4f7a89f1c --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log @@ -0,0 +1,301 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735706664.663046]: resetting environment +[INFO] [1735706664.669216]: cleared current plan +[INFO] [1735706664.670038]: moving to home +[INFO] [1735706665.672523]: reached home +[INFO] [1735706665.673874]: reset action history +[INFO] [1735706668.384475]: environment reset +Starting episode 0 +[INFO] [1735706668.389221]: resetting environment +[INFO] [1735706668.395132]: cleared current plan +[INFO] [1735706668.396672]: moving to home +[INFO] [1735706669.398592]: reached home +[INFO] [1735706669.399855]: reset action history +[INFO] [1735706672.107597]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735706676.726071]: resetting environment +[INFO] [1735706676.731868]: cleared current plan +[INFO] [1735706676.732646]: moving to home +[INFO] [1735706677.735011]: reached home +[INFO] [1735706677.736268]: reset action history +[INFO] [1735706680.450655]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/episode_rosbags/episode_0_2024-12-31-23-44-40.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1769082.log +started roslaunch server http://158.130.50.37:38045/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1769380] +process[print_text-2]: started with pid [1769404] +process[rosbag_record-3]: started with pid [1769405] +started bagging! +For topic gripper_width: timestamp difference is 100454275 for nearest: 1735706694719977562 - target: 1735706694619523287 at idx 179 +For topic gripper_width: timestamp difference is 101420062 for nearest: 1735706696019899505 - target: 1735706695918479443 at idx 190 +For topic gripper_width: timestamp difference is 102963444 for nearest: 1735706697819971796 - target: 1735706697717008352 at idx 215 +[INFO] [1735706715.509216]: Storing episode... +[rosbag_record-3] killing on exit[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735706716.322256]: Stored episode 1. +[INFO] [1735706716.322485]: Saving video... +[INFO] [1735706716.511070]: Video saved! +Starting episode 1 +[INFO] [1735706716.525409]: resetting environment +[INFO] [1735706716.532987]: cleared current plan +[INFO] [1735706716.533231]: moving to home +Move to target timeout: 15.001429080963135. Trans error: 0.0075681465677917. Rot error: 1.1976080955949815 +Move to target timeout: 15.099565744400024. Trans error: 0.005587152705862385. Rot error: 0.9468543956861867 +[INFO] [1735706747.636561]: reached home +[INFO] [1735706747.636997]: reset action history +[INFO] [1735706750.348117]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735706765.624948]: resetting environment +[INFO] [1735706765.628547]: cleared current plan +[INFO] [1735706765.628741]: moving to home +Move to target timeout: 15.000636577606201. Trans error: 0.005397010128945112. Rot error: 0.5659157845792888 +Move to target timeout: 15.09976863861084. Trans error: 0.005338925860442211. Rot error: 1.3136386765760355 +[INFO] [1735706796.730890]: reached home +[INFO] [1735706796.731295]: reset action history +[INFO] [1735706799.439529]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/episode_rosbags/episode_1_2024-12-31-23-46-39.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1769082.log +started roslaunch server http://158.130.50.37:34955/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1769532] +process[print_text-5]: started with pid [1769556] +process[rosbag_record-6]: started with pid [1769557] +started bagging! +For topic gripper_width: timestamp difference is 101729434 for nearest: 1735706815819944423 - target: 1735706815718214989 at idx 225 +For topic gripper_width: timestamp difference is 105968498 for nearest: 1735706825419908069 - target: 1735706825313939571 at idx 249 +[INFO] [1735706834.303024]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735706835.024516]: Stored episode 2. +[INFO] [1735706835.024806]: Saving video... +[INFO] [1735706835.648527]: Video saved! +Starting episode 2 +[INFO] [1735706835.666008]: resetting environment +[INFO] [1735706835.678816]: cleared current plan +[INFO] [1735706835.679078]: moving to home +[WARN] [1735706835.671350]: Plan exhausted +Move to target timeout: 15.002738952636719. Trans error: 0.007834541611373425. Rot error: 1.2421536891002385 +Move to target timeout: 15.096193552017212. Trans error: 0.005578317346684229. Rot error: 0.9217834318828864 +[INFO] [1735706866.782914]: reached home +[INFO] [1735706866.785307]: reset action history +[INFO] [1735706869.557898]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735706879.922664]: resetting environment +[INFO] [1735706879.932470]: cleared current plan +[INFO] [1735706879.932658]: moving to home +[INFO] [1735706880.933252]: reached home +[INFO] [1735706880.933923]: reset action history +[INFO] [1735706883.642591]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/episode_rosbags/episode_2_2024-12-31-23-48-03.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1769082.log +started roslaunch server http://158.130.50.37:44529/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1769972] +process[print_text-8]: started with pid [1769973] +process[rosbag_record-9]: started with pid [1769974] +started bagging! +preempting episode +[INFO] [1735706916.826453]: Storing episode... +[rosbag_record-9] killing on exit[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735706917.769305]: Stored episode 3. +[INFO] [1735706917.770202]: Saving video... +[WARN] [1735706918.036590]: Plan exhausted +[WARN] [1735706918.078145]: Plan exhausted +[WARN] [1735706918.116393]: Plan exhausted +[WARN] [1735706918.156583]: Plan exhausted +[WARN] [1735706918.197957]: Plan exhausted +[WARN] [1735706918.237188]: Plan exhausted +[WARN] [1735706918.276784]: Plan exhausted +[WARN] [1735706918.319906]: Plan exhausted +[WARN] [1735706918.356466]: Plan exhausted +[WARN] [1735706918.396608]: Plan exhausted +[INFO] [1735706918.370517]: Video saved! +Starting episode 3 +[INFO] [1735706918.410143]: resetting environment +[INFO] [1735706918.423990]: cleared current plan +[INFO] [1735706918.424609]: moving to home +[INFO] [1735706937.733062]: reached home +[INFO] [1735706937.741421]: reset action history +[INFO] [1735706940.466014]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735706947.385038]: resetting environment +[INFO] [1735706947.413736]: cleared current plan +[INFO] [1735706947.416544]: moving to home +[INFO] [1735706948.421001]: reached home +[INFO] [1735706948.425340]: reset action history +[INFO] [1735706951.146247]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/episode_rosbags/episode_3_2024-12-31-23-49-11.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1769082.log +started roslaunch server http://158.130.50.37:37889/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1770115] +process[print_text-11]: started with pid [1770116] +process[rosbag_record-12]: started with pid [1770140] +started bagging! +For topic gripper_width: timestamp difference is 100141203 for nearest: 1735706983319966709 - target: 1735706983219825506 at idx 249 +[INFO] [1735706985.926586]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[WARN] [1735706987.279633]: Plan exhausted +[WARN] [1735706987.316560]: Plan exhausted +[INFO] [1735706986.715647]: Stored episode 4. +[INFO] [1735706986.720538]: Saving video... +[INFO] [1735706987.287030]: Video saved! +Starting episode 4 +[INFO] [1735706987.343850]: resetting environment +[INFO] [1735706987.370931]: cleared current plan +[INFO] [1735706987.374846]: moving to home +[WARN] [1735706987.356390]: Plan exhausted +Move to target timeout: 15.003847599029541. Trans error: 0.007557788398116827. Rot error: 1.2057280608064258 +[INFO] [1735707011.783800]: reached home +[INFO] [1735707011.787890]: reset action history +[INFO] [1735707014.526989]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735707028.222369]: resetting environment +[INFO] [1735707028.252649]: cleared current plan +[INFO] [1735707028.258865]: moving to home +Move to target timeout: 15.001442909240723. Trans error: 0.005209829690532132. Rot error: 1.1128904704423823 +[INFO] [1735707044.466608]: reached home +[INFO] [1735707044.469505]: reset action history +[INFO] [1735707047.242012]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/episode_rosbags/episode_4_2024-12-31-23-50-47.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1769082.log +started roslaunch server http://158.130.50.37:41393/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1770264] +process[print_text-14]: started with pid [1770265] +process[rosbag_record-15]: started with pid [1770289] +started bagging! +For topic gripper_width: timestamp difference is -105029055 for nearest: 1735707059253274253 - target: 1735707059358303308 at idx 164 +For topic gripper_width: timestamp difference is 100034953 for nearest: 1735707065719972850 - target: 1735707065619937897 at idx 249 +For topic gripper_width: timestamp difference is 100802995 for nearest: 1735707066753283359 - target: 1735707066652480364 at idx 249 +For topic gripper_width: timestamp difference is 105017696 for nearest: 1735707071919940982 - target: 1735707071814923286 at idx 249 +For topic gripper_width: timestamp difference is -159493927 for nearest: 1735707074253277060 - target: 1735707074412770987 at idx 249 +For topic gripper_width: timestamp difference is -123791411 for nearest: 1735707077153269098 - target: 1735707077277060509 at idx 249 +[INFO] [1735707082.058980]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[WARN] [1735707083.397431]: Plan exhausted +[INFO] [1735707082.847307]: Stored episode 5. +[INFO] [1735707082.852167]: Saving video... +[INFO] [1735707083.426228]: Video saved! +[INFO] [1735707083.493599]: resetting environment +[INFO] [1735707083.507851]: cleared current plan +[INFO] [1735707083.508088]: moving to home +[WARN] [1735707083.439862]: Plan exhausted +[WARN] [1735707083.476385]: Plan exhausted +Move to target timeout: 15.001688003540039. Trans error: 0.00791110284626484. Rot error: 1.2722322330275395 +Move to target timeout: 15.093971252441406. Trans error: 0.00553666208000334. Rot error: 0.9160281085193275 +[INFO] [1735707114.619320]: reached home +[INFO] [1735707114.624546]: reset action history +[INFO] [1735707117.436914]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/requirements.txt b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-metadata.json b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..99008b38f2d423568c355bc7cee88dce9cc6d4fb --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T04:44:20.186033", + "startedAt": "2025-01-01T04:44:19.867291", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.4945, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.02, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 609.4676246643066 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..df88cab9385305a5f68a5e4b3d7fe1a71064a406 --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "4f97d134fa70e07b1df3ca6dbef126a8358e555942b7d6a536d5198e26a620b6", "size": 1433247, "path": "media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4"}, "global_step": 658, "_timestamp": 1735707121.78647, "_runtime": 461.90927505493164, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "92c79785e0b0a9901616e727807903d0eb95a515072c2f2cfc59c47a6cbbe252", "size": 1436301, "path": "media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4"}, "eval/num_success": 0.0, "episode": 25.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "942af39625588acf70b9d6f37b8e15e201d91f3e4c22855a7ac3a17c901cc2e4", "size": 1462800, "path": "media/videos/eval/2_eval_2_942af39625588acf70b9.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "511ada0368931862d4528110e7fa8f28fcf03272a520b43d6a79c1975f2ad0be", "size": 1450058, "path": "media/videos/eval/3_eval_3_511ada0368931862d452.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "f5faf9c54c3583c3620ccbaab96c58a003ca978ad55404a5d50e1b4f2764ce25", "size": 1481514, "path": "media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://su9ftb2hcizoe8w88x0dwg4x1z9r6jh3rvfiqnbsq71yy0g29od10ubvk6oj7eq1k7z4bocga2rkziu2hg9e1bugc9bmnfmz2wicbiujulbkt4w7qqlevjjn1bep5d51/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://8x2j98w1qniof12rvldmsbmtzxgxhdde762901zb23ob6msccmqo2o1g621v3t25amr1yht6ox9apx8mn6yg9d94455jwk4szhq7h1rlpmcn4hqxfy4cwf0rjhg1jqaa:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 10, "total_num_episodes": 25, "total_success_rate": 0.4, "total_success_rate_lb_0.01": 0.19247555738877542, "total_success_rate_ub_0.01": 0.6353677510944474, "total_success_rate_lb_0.025": 0.23085720842499102, "total_success_rate_ub_0.025": 0.6117631792798629, "total_success_rate_lb_0.05": 0.25551603739382933, "total_success_rate_ub_0.05": 0.5599231943368275, "total_success_rate_lb_0.1": 0.27121893693611987, "total_success_rate_ub_0.1": 0.5436965897591937, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "1ee4b661c7d054c174ffad379cda12f6b09a9bff39f4c4f0b1a2d5df5494c197", "size": 509, "artifact_path": "wandb-client-artifact://uwb90zj7i4ykze9qehrujsirdvr2getrsnxuqq9ywlvzaak3i3v3iqn3ucqpilkfnd8lmqkjisy9eaw2zxeqbrn0uvcdngji3f36f0npbusa5tbg7mz5mm6dudlf4nrf/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://m0ux21nd22mtde86c9wh2jpbcipdh1aou26ek1bd5dijxamrdysizcfqz204j87cjgh2dl5cp4c2t10gj9bl82cuxuv1w9bgml88fsjz3jfzbcjyfj7snpi09m2jr4fr:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 463}, "eval/total_success_rate": 0.4000000059604645, "eval/total_success_rate_lb_0.01": 0.1924755573272705, "eval/total_success_rate_ub_0.01": 0.635367751121521, "eval/total_success_rate_lb_0.025": 0.23085720837116241, "eval/total_success_rate_ub_0.025": 0.6117631793022156, "eval/total_success_rate_lb_0.05": 0.25551605224609375, "eval/total_success_rate_ub_0.05": 0.5599231719970703, "eval/total_success_rate_lb_0.1": 0.2712189257144928, "eval/total_success_rate_ub_0.1": 0.5436965823173523} \ No newline at end of file diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/logs/debug-internal.log b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..291e9eec1607092ffc9a0b56ad60a06e89b5072e --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/logs/debug-internal.log @@ -0,0 +1,1055 @@ +2024-12-31 23:44:19,877 INFO StreamThr :1769274 [internal.py:wandb_internal():85] W&B internal server running at pid: 1769274, started at: 2024-12-31 23:44:19.877147 +2024-12-31 23:44:19,878 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status +2024-12-31 23:44:19,889 INFO WriterThread:1769274 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/run-fkfuijgf.wandb +2024-12-31 23:44:19,890 DEBUG SenderThread:1769274 [sender.py:send():379] send: header +2024-12-31 23:44:19,897 DEBUG SenderThread:1769274 [sender.py:send():379] send: run +2024-12-31 23:44:20,105 INFO SenderThread:1769274 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files +2024-12-31 23:44:20,105 INFO SenderThread:1769274 [sender.py:_start_run_threads():1188] run started: fkfuijgf with start time 1735706659.877195 +2024-12-31 23:44:20,120 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 23:44:20,120 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: check_version +2024-12-31 23:44:20,165 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 23:44:20,170 DEBUG HandlerThread:1769274 [system_info.py:__init__():26] System info init +2024-12-31 23:44:20,170 DEBUG HandlerThread:1769274 [system_info.py:__init__():41] System info init done +2024-12-31 23:44:20,170 INFO HandlerThread:1769274 [system_monitor.py:start():194] Starting system monitor +2024-12-31 23:44:20,170 INFO SystemMonitor:1769274 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 23:44:20,170 INFO HandlerThread:1769274 [system_monitor.py:probe():214] Collecting system info +2024-12-31 23:44:20,171 INFO SystemMonitor:1769274 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 23:44:20,172 INFO SystemMonitor:1769274 [interfaces.py:start():188] Started disk monitoring +2024-12-31 23:44:20,173 INFO SystemMonitor:1769274 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 23:44:20,175 INFO SystemMonitor:1769274 [interfaces.py:start():188] Started memory monitoring +2024-12-31 23:44:20,175 INFO SystemMonitor:1769274 [interfaces.py:start():188] Started network monitoring +2024-12-31 23:44:20,185 DEBUG HandlerThread:1769274 [system_info.py:probe():152] Probing system +2024-12-31 23:44:20,187 DEBUG HandlerThread:1769274 [system_info.py:_probe_git():137] Probing git +2024-12-31 23:44:20,192 DEBUG HandlerThread:1769274 [system_info.py:_probe_git():145] Probing git done +2024-12-31 23:44:20,192 DEBUG HandlerThread:1769274 [system_info.py:probe():200] Probing system done +2024-12-31 23:44:20,192 DEBUG HandlerThread:1769274 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T04:44:20.186033', 'startedAt': '2025-01-01T04:44:19.867291', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.4945, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.02, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 609.4676246643066}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 23:44:20,192 INFO HandlerThread:1769274 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 23:44:20,192 INFO HandlerThread:1769274 [system_monitor.py:probe():227] Publishing system info +2024-12-31 23:44:20,192 DEBUG HandlerThread:1769274 [system_info.py:_save_code():44] Saving code +2024-12-31 23:44:20,193 DEBUG HandlerThread:1769274 [system_info.py:_save_code():67] Saving code done +2024-12-31 23:44:20,193 DEBUG HandlerThread:1769274 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 23:44:20,255 DEBUG HandlerThread:1769274 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 23:44:20,255 INFO HandlerThread:1769274 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 23:44:20,259 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:44:20,259 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 23:44:20,259 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 23:44:20,259 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 23:44:20,488 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/3et9molp-wandb-metadata.json +2024-12-31 23:44:20,500 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 23:44:20,501 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: python_packages +2024-12-31 23:44:20,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:44:20,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:20,502 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:44:20,505 INFO HandlerThread:1769274 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/tb" + save: true +} +control { + relay_id: "8e3059a3c81d41b2beda0ecfc9f46d93" +} +_info { + stream_id: "fkfuijgf" +} + +2024-12-31 23:44:20,506 DEBUG HandlerThread:1769274 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 23:44:20,542 INFO wandb-upload_2:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/42d55c61-diff.patch +2024-12-31 23:44:20,543 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/m6e1crnu-code/FISH/eval_robot.py +2024-12-31 23:44:20,590 DEBUG SenderThread:1769274 [sender.py:send():379] send: telemetry +2024-12-31 23:44:20,590 DEBUG SenderThread:1769274 [sender.py:send():379] send: tbrecord +2024-12-31 23:44:20,590 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:44:20,590 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 with policy live +2024-12-31 23:44:20,786 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/0rjdmvik-events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:44:21,107 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:21,107 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/code/FISH/eval_robot.py +2024-12-31 23:44:21,107 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:44:21,107 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/diff.patch +2024-12-31 23:44:21,107 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/requirements.txt +2024-12-31 23:44:21,107 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-metadata.json +2024-12-31 23:44:21,108 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/code/FISH +2024-12-31 23:44:21,108 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/code +2024-12-31 23:44:21,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:22,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:23,106 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:23,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:24,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:24,662 DEBUG SenderThread:1769274 [sender.py:send():379] send: config +2024-12-31 23:44:25,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:44:25,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:26,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:27,107 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:27,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:28,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:29,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:30,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:44:30,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:31,108 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:31,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:32,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:33,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:34,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:35,109 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:35,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:44:35,501 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:44:35,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:35,653 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:44:36,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:37,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:38,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:39,112 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:39,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:40,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:40,661 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:44:41,506 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:42,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:43,113 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:43,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:44,506 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:45,506 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:46,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:44:46,508 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:47,506 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:48,507 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:49,507 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:50,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:44:50,501 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:44:50,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:51,507 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:51,584 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:44:52,115 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/config.yaml +2024-12-31 23:44:52,507 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:53,507 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:54,508 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:55,510 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:56,509 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:57,069 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:44:57,117 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:57,509 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:58,509 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:44:59,117 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:44:59,509 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:00,509 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:01,510 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:02,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:02,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:03,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:04,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:05,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:45:05,501 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:45:05,549 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:06,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:07,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:07,622 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:08,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:09,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:10,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:11,512 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:12,512 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:13,503 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:13,512 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:14,512 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:15,557 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:16,512 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:16,522 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:45:16,522 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 with policy now +2024-12-31 23:45:16,565 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:45:16,566 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:45:16,566 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:45:16,567 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:45:16,791 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/seuf4ki2-media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 +2024-12-31 23:45:17,122 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:45:17,122 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:45:17,122 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 +2024-12-31 23:45:17,122 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media +2024-12-31 23:45:17,123 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos +2024-12-31 23:45:17,123 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval +2024-12-31 23:45:17,423 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/1unwdjkl-media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 +2024-12-31 23:45:17,512 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:18,513 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:18,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:19,123 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:45:19,513 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:20,175 DEBUG SystemMonitor:1769274 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 23:45:20,176 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:45:20,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:45:20,501 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:45:20,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:21,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:22,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:23,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:23,619 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:24,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:25,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:26,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:27,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:28,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:29,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:29,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:30,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:31,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:32,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:33,127 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:45:33,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:34,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:34,577 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:35,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:45:35,502 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:45:35,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:36,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:37,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:38,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:39,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:39,603 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:40,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:41,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:42,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:43,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:44,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:45,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:45,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:46,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:47,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:48,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:49,132 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:45:49,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:50,177 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:45:50,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:45:50,501 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:45:50,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:50,563 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:51,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:52,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:53,133 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:45:53,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:54,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:55,518 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:56,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:45:56,518 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:57,518 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:58,518 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:45:59,518 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:00,518 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:01,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:02,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:02,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:03,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:04,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:05,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:46:05,501 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:46:05,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:06,138 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:46:06,413 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/erf1smsg-events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:46:06,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:07,138 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:46:07,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:07,639 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:08,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:09,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:10,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:11,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:12,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:13,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:13,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:14,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:15,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:16,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:17,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:18,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:18,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:19,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:20,177 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:46:20,501 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:46:20,502 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:46:20,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:21,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:22,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:23,143 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:46:23,521 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:23,645 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:24,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:25,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:26,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:27,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:28,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:29,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:29,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:30,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:31,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:32,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:33,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:34,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:34,523 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:35,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:46:35,502 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:46:35,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:36,523 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:37,147 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:46:37,523 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:38,523 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:39,148 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:46:39,523 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:39,639 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:40,523 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:41,149 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:46:41,523 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:42,526 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:43,528 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:44,528 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:45,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:45,528 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:46,528 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:47,528 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:48,528 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:49,528 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:50,178 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:46:50,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:46:50,502 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:46:50,549 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:50,596 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:51,529 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:52,529 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:53,529 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:54,529 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:55,529 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:55,833 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:46:56,530 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:57,154 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:46:57,530 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:58,530 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:46:59,530 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:00,530 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:01,507 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:01,531 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:02,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:03,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:04,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:05,502 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:47:05,502 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:47:05,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:06,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:06,655 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:07,157 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:47:07,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:08,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:09,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:10,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:11,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:12,507 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:12,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:13,533 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:14,533 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:15,540 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:15,693 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:47:15,694 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:47:15,694 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 with policy now +2024-12-31 23:47:15,695 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:47:15,695 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:47:15,695 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:47:15,968 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/zwe35cme-media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 +2024-12-31 23:47:16,160 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:47:16,160 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 +2024-12-31 23:47:16,160 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval +2024-12-31 23:47:16,448 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/ac40026d-media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 +2024-12-31 23:47:16,542 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:17,160 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:47:17,545 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:17,697 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:18,161 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:47:18,546 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:19,544 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:20,178 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:47:20,505 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:47:20,505 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:47:20,549 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:21,548 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:22,550 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:23,550 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:23,587 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:24,548 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:25,553 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:26,549 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:27,554 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:28,549 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:29,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:29,555 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:30,552 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:31,554 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:32,554 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:33,165 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:47:33,556 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:34,556 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:34,729 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:35,511 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:47:35,511 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:47:35,555 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:36,554 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:37,555 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:38,559 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:39,560 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:40,517 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:40,557 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:41,559 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:42,560 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:43,563 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:44,562 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:45,565 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:45,781 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:46,563 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:47,170 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:47:47,564 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:48,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:49,170 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:47:49,605 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:50,178 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:47:50,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:47:50,505 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:47:50,565 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:51,171 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:47:51,564 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:51,566 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:52,565 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:53,565 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:54,565 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:55,565 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:56,566 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:57,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:47:57,566 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:58,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:47:59,566 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:00,174 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:48:00,432 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/3oubdk4l-events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:48:00,566 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:01,047 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:48:01,048 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:48:01,048 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:48:01,174 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:48:01,175 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:48:01,566 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:02,569 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:03,048 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:03,175 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:48:03,570 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:04,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:05,176 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:48:05,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:48:05,505 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:48:05,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:06,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:07,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:08,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:08,582 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:09,567 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:10,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:11,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:12,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:13,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:14,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:14,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:15,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:16,569 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:17,568 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:18,569 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:19,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:19,569 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:20,179 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:48:20,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:48:20,505 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:48:20,570 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:21,570 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:22,570 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:23,570 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:24,570 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:24,620 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:25,571 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:26,571 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:27,571 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:28,572 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:29,572 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:30,515 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:30,572 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:31,572 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:32,572 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:33,573 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:34,572 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:35,504 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:48:35,505 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:48:35,572 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:35,579 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:36,573 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:37,576 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:38,405 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:48:38,405 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 with policy now +2024-12-31 23:48:38,408 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:48:38,408 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:48:38,409 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:48:38,409 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:48:38,574 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:38,680 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/frhyj3mk-media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 +2024-12-31 23:48:39,187 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:48:39,187 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:48:39,187 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 +2024-12-31 23:48:39,188 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval +2024-12-31 23:48:39,496 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/sh64xtaa-media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 +2024-12-31 23:48:39,576 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:40,187 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:48:40,577 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:41,188 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:48:41,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:41,578 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:42,577 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:43,580 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:44,579 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:45,583 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:46,584 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:47,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:47,582 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:48,581 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:49,583 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:50,179 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:48:50,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:48:50,514 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:48:50,582 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:51,583 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:52,582 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:52,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:53,584 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:54,584 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:55,584 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:56,585 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:57,587 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:57,734 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:48:58,590 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:48:59,194 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:48:59,591 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:00,587 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:01,589 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:02,594 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:03,196 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:03,518 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:03,598 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:04,599 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:05,509 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:49:05,510 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:49:05,601 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:06,601 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:07,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:07,908 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:49:07,909 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:49:07,909 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:49:08,197 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:49:08,198 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:49:08,444 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/ot7fldun-events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:49:08,603 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:09,198 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:09,469 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:09,601 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:10,605 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:11,199 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:11,601 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:12,601 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:13,199 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:13,601 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:14,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:15,408 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:15,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:16,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:17,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:18,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:19,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:20,180 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:49:20,509 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:49:20,510 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:49:20,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:20,629 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:21,602 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:22,603 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:23,603 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:24,603 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:25,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:26,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:26,603 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:27,603 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:28,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:29,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:30,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:31,519 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:31,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:32,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:33,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:34,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:35,510 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:49:35,510 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:49:35,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:36,576 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:36,604 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:37,606 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:38,606 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:39,606 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:40,606 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:41,607 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:42,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:42,607 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:43,607 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:44,609 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:45,210 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:45,609 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:46,620 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:47,211 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:47,331 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:49:47,332 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_511ada0368931862d452.mp4 with policy now +2024-12-31 23:49:47,335 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:49:47,335 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:49:47,335 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:49:47,357 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:49:47,605 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/r9n8aqwr-media/videos/eval/3_eval_3_511ada0368931862d452.mp4 +2024-12-31 23:49:47,623 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:48,211 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:48,212 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:49:48,212 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/3_eval_3_511ada0368931862d452.mp4 +2024-12-31 23:49:48,212 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval +2024-12-31 23:49:48,376 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:48,568 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/p0kbzqpj-media/videos/eval/3_eval_3_511ada0368931862d452.mp4 +2024-12-31 23:49:48,611 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:49,212 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:49,612 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:50,180 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:49:50,212 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:49:50,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:49:50,520 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:49:50,620 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:51,615 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:52,613 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:53,600 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:53,614 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:54,614 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:55,623 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:56,623 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:57,624 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:58,622 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:49:59,526 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:49:59,629 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:00,622 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:01,628 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:02,628 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:03,216 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:50:03,624 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:04,628 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:05,429 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:05,516 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:50:05,516 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:50:05,624 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:06,628 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:07,625 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:08,629 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:09,634 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:10,530 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:10,642 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:11,640 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:12,640 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:13,219 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:50:13,650 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:14,641 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:15,646 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:16,530 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:16,644 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:17,221 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:50:17,648 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:18,644 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:19,646 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:20,181 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:50:20,522 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:50:20,522 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:50:20,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:21,651 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:21,672 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:22,654 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:23,655 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:24,656 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:25,658 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:26,661 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:27,533 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:27,658 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:28,224 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:50:28,663 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:29,003 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:50:29,003 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:50:29,003 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:50:29,225 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:50:29,225 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:50:29,225 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:50:29,476 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/iq5zua9g-events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:50:29,661 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:30,666 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:31,666 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:32,666 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:33,533 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:33,664 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:34,666 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:35,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:50:35,520 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:50:35,667 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:36,665 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:37,667 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:38,607 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:38,668 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:39,671 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:40,669 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:41,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:42,676 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:43,675 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:44,467 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:44,668 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:45,230 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:50:45,672 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:46,676 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:47,230 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:50:47,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:48,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:49,231 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:50:49,510 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:49,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:50,181 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:50:50,513 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:50:50,514 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:50:50,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:51,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:52,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:53,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:54,653 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:50:54,670 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:55,671 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:56,671 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:57,671 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:58,671 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:50:59,671 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:00,485 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:00,674 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:01,235 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:01,672 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:02,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:03,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:04,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:05,513 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:51:05,514 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:51:05,618 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:05,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:06,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:07,238 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:07,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:08,674 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:09,239 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:09,673 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:10,674 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:11,531 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:11,674 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:12,674 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:13,240 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:13,675 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:14,675 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:15,241 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:15,675 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:16,532 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:16,675 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:17,677 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:18,676 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:19,242 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:19,676 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:20,182 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:51:20,514 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:51:20,514 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:51:20,676 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:21,665 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:21,676 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:22,692 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:23,244 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:23,491 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:51:23,491 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 with policy now +2024-12-31 23:51:23,493 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:51:23,494 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:51:23,494 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:51:23,494 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:51:23,682 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:23,780 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/tfgiibxx-media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 +2024-12-31 23:51:24,244 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:24,244 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:51:24,245 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 +2024-12-31 23:51:24,245 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval +2024-12-31 23:51:24,554 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/3cg14ctk-media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 +2024-12-31 23:51:24,691 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:25,245 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:25,688 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:26,245 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:26,691 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:27,537 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:27,698 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:28,706 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:29,702 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:30,702 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:31,710 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:32,548 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:32,710 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:33,714 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:34,717 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:35,520 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:51:35,520 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:51:35,710 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:36,721 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:37,580 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:37,714 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:38,717 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:39,249 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:39,716 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:40,723 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:41,727 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:42,729 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:43,541 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:43,732 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:44,734 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:45,732 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:46,727 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:47,728 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:48,547 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:48,726 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:49,729 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:50,182 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:51:50,526 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:51:50,526 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:51:50,738 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:51,732 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:52,731 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:53,612 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:53,731 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:54,734 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:55,255 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:55,735 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:56,740 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:57,256 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:57,732 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:58,737 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:51:59,256 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:51:59,477 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:51:59,732 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:00,731 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:01,110 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:52:01,110 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:01,110 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:01,257 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:52:01,257 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:52:01,257 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:52:01,300 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:52:01,300 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:52:01,512 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:52:01,520 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/birw3f9s-events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:52:01,731 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:01,767 INFO wandb-upload_0:1769274 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp8i279p53 +2024-12-31 23:52:01,890 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:52:01,900 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:52:02,128 INFO SenderThread:1769274 [sender.py:send_request_log_artifact():1525] logged artifact run-fkfuijgf-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIxNDAwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3OTYzMQ==', 'latestArtifact': None}} +2024-12-31 23:52:02,128 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:52:02,128 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2024-12-31 23:52:02,129 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:52:02,129 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,129 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,129 DEBUG SenderThread:1769274 [sender.py:send():379] send: telemetry +2024-12-31 23:52:02,129 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,130 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,130 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,130 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,130 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,132 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,132 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,133 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,133 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,133 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,133 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,134 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,134 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,134 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,134 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,135 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,135 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,135 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,135 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,136 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,136 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:02,136 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:02,136 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:52:02,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:52:02,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:52:02,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json +2024-12-31 23:52:02,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:52:02,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media +2024-12-31 23:52:02,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval +2024-12-31 23:52:02,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table +2024-12-31 23:52:02,328 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/y2b70qk6-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:52:02,496 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/wy349f5u-media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json +2024-12-31 23:52:02,548 INFO wandb-upload_2:1769274 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp58yspos_ +2024-12-31 23:52:02,607 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/gue7j1tz-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:52:02,731 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:03,040 INFO SenderThread:1769274 [sender.py:send_request_log_artifact():1525] logged artifact run-fkfuijgf-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIxNDAxMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3OTYzMg==', 'latestArtifact': None}} +2024-12-31 23:52:03,040 DEBUG SenderThread:1769274 [sender.py:send():379] send: files +2024-12-31 23:52:03,040 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json with policy now +2024-12-31 23:52:03,040 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:52:03,040 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:03,041 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:03,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:52:03,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:52:03,294 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/5nmobrxv-media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json +2024-12-31 23:52:03,645 DEBUG SenderThread:1769274 [sender.py:send():379] send: exit +2024-12-31 23:52:03,645 INFO SenderThread:1769274 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 23:52:03,646 INFO SenderThread:1769274 [sender.py:send_exit():588] handling runtime: 463 +2024-12-31 23:52:03,646 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:03,646 INFO SenderThread:1769274 [sender.py:send_exit():594] send defer +2024-12-31 23:52:03,646 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:03,646 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 23:52:03,646 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:03,646 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 23:52:03,646 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 1 +2024-12-31 23:52:03,647 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:03,647 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 23:52:03,647 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:03,647 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 23:52:03,647 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 2 +2024-12-31 23:52:03,647 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:03,647 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 23:52:03,647 INFO HandlerThread:1769274 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 23:52:03,647 DEBUG SystemMonitor:1769274 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 23:52:03,647 DEBUG SystemMonitor:1769274 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 23:52:03,648 INFO HandlerThread:1769274 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 23:52:03,648 INFO HandlerThread:1769274 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 23:52:03,699 INFO HandlerThread:1769274 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 23:52:03,699 INFO HandlerThread:1769274 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 23:52:03,699 INFO HandlerThread:1769274 [interfaces.py:finish():200] Joined network monitor +2024-12-31 23:52:03,700 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:03,700 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 23:52:03,700 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 3 +2024-12-31 23:52:03,700 DEBUG SenderThread:1769274 [sender.py:send():379] send: stats +2024-12-31 23:52:03,700 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:03,700 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 23:52:03,700 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:03,700 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 23:52:03,700 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 4 +2024-12-31 23:52:03,701 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:03,701 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 23:52:04,258 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:52:05,259 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:52:10,115 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:52:10,116 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:10,116 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 23:52:10,116 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:10,117 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:10,117 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 23:52:10,118 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 5 +2024-12-31 23:52:10,118 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:52:10,118 DEBUG SenderThread:1769274 [sender.py:send():379] send: history +2024-12-31 23:52:10,118 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:52:10,118 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:10,118 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 23:52:10,119 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:10,119 DEBUG SenderThread:1769274 [sender.py:send():379] send: summary +2024-12-31 23:52:10,122 INFO SenderThread:1769274 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:52:10,123 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:10,123 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 23:52:10,123 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 6 +2024-12-31 23:52:10,123 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:10,123 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 23:52:10,123 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:10,123 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 23:52:10,126 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:10,251 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 7 +2024-12-31 23:52:10,251 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:10,251 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 23:52:10,251 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:10,251 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 23:52:10,261 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:52:10,261 INFO Thread-12 :1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/config.yaml +2024-12-31 23:52:10,647 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:52:11,972 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 8 +2024-12-31 23:52:11,972 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:52:11,972 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:11,973 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 23:52:11,973 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:11,973 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 23:52:11,973 INFO SenderThread:1769274 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 23:52:11,975 INFO SenderThread:1769274 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 23:52:11,977 INFO SenderThread:1769274 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 23:52:11,990 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 9 +2024-12-31 23:52:11,991 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:11,991 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 23:52:11,991 DEBUG SenderThread:1769274 [sender.py:send():379] send: artifact +2024-12-31 23:52:12,157 INFO SenderThread:1769274 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIwNzU2Mg==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDIwNzU2Mg==', 'versionIndex': 401}}} +2024-12-31 23:52:12,157 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:12,157 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 23:52:12,157 INFO SenderThread:1769274 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 23:52:12,262 INFO SenderThread:1769274 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:52:12,262 INFO SenderThread:1769274 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files +2024-12-31 23:52:12,262 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/config.yaml config.yaml +2024-12-31 23:52:12,263 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:52:12,263 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json wandb-summary.json +2024-12-31 23:52:12,264 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/requirements.txt requirements.txt +2024-12-31 23:52:12,265 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-metadata.json wandb-metadata.json +2024-12-31 23:52:12,265 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log output.log +2024-12-31 23:52:12,265 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/diff.patch diff.patch +2024-12-31 23:52:12,265 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 23:52:12,269 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 media/videos/eval/1_eval_1_92c79785e0b0a9901616.mp4 +2024-12-31 23:52:12,270 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 media/videos/eval/4_eval_4_f5faf9c54c3583c3620c.mp4 +2024-12-31 23:52:12,270 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 media/videos/eval/2_eval_2_942af39625588acf70b9.mp4 +2024-12-31 23:52:12,271 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 media/videos/eval/0_eval_0_4f97d134fa70e07b1df3.mp4 +2024-12-31 23:52:12,271 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/videos/eval/3_eval_3_511ada0368931862d452.mp4 media/videos/eval/3_eval_3_511ada0368931862d452.mp4 +2024-12-31 23:52:12,271 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 23:52:12,271 INFO SenderThread:1769274 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json media/table/eval/total_success_rate_ci_6_1ee4b661c7d054c174ff.table.json +2024-12-31 23:52:12,271 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 10 +2024-12-31 23:52:12,271 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:12,271 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 23:52:12,272 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:12,272 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 23:52:12,272 INFO SenderThread:1769274 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:52:12,461 INFO wandb-upload_1:1769274 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/config.yaml +2024-12-31 23:52:12,463 INFO wandb-upload_2:1769274 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/wandb-summary.json +2024-12-31 23:52:12,499 INFO wandb-upload_4:1769274 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/output.log +2024-12-31 23:52:12,505 INFO wandb-upload_0:1769274 [upload_job.py:push():130] Uploaded file /tmp/tmp07m76vnewandb/mdgf2ixm-events.out.tfevents.1735706660.leonmkim-ROG-Strix-G15CS-G15CS.1769082.0 +2024-12-31 23:52:12,509 INFO wandb-upload_3:1769274 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/files/requirements.txt +2024-12-31 23:52:12,647 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:52:12,648 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:52:12,709 INFO Thread-11 (_thread_body):1769274 [sender.py:transition_state():614] send defer: 11 +2024-12-31 23:52:12,709 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:12,709 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 23:52:12,709 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:12,709 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 23:52:12,710 INFO SenderThread:1769274 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 23:52:12,711 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 12 +2024-12-31 23:52:12,711 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:12,711 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 23:52:12,711 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:12,711 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 23:52:12,711 INFO SenderThread:1769274 [file_stream.py:finish():601] file stream finish called +2024-12-31 23:52:12,896 INFO SenderThread:1769274 [file_stream.py:finish():605] file stream finish is done +2024-12-31 23:52:12,896 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 13 +2024-12-31 23:52:12,896 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:12,896 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 23:52:12,896 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:12,896 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 23:52:12,897 INFO SenderThread:1769274 [sender.py:transition_state():614] send defer: 14 +2024-12-31 23:52:12,897 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:52:12,897 INFO HandlerThread:1769274 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 23:52:12,897 DEBUG SenderThread:1769274 [sender.py:send():379] send: final +2024-12-31 23:52:12,897 DEBUG SenderThread:1769274 [sender.py:send():379] send: footer +2024-12-31 23:52:12,897 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: defer +2024-12-31 23:52:12,897 INFO SenderThread:1769274 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 23:52:12,898 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:52:12,898 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 23:52:12,898 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:52:12,898 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:52:12,898 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 23:52:12,898 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: server_info +2024-12-31 23:52:12,899 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 23:52:12,901 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:12,950 DEBUG SenderThread:1769274 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:52:12,950 INFO MainThread:1769274 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 23:52:12,950 INFO MainThread:1769274 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 23:52:12,951 INFO MainThread:1769274 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 23:52:12,951 DEBUG HandlerThread:1769274 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 23:52:12,951 INFO HandlerThread:1769274 [handler.py:finish():882] shutting down handler +2024-12-31 23:52:13,898 INFO WriterThread:1769274 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/run-fkfuijgf.wandb +2024-12-31 23:52:13,950 INFO SenderThread:1769274 [sender.py:finish():1615] shutting down sender +2024-12-31 23:52:13,950 INFO SenderThread:1769274 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:52:13,950 INFO SenderThread:1769274 [file_pusher.py:join():175] waiting for file pusher diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/logs/debug.log b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a6e1076d8261a135e7f9c06f57bf32d0b05bc914 --- /dev/null +++ b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-31 23:44:19,869 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 23:44:19,869 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Configure stats pid to 1769082 +2024-12-31 23:44:19,869 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 23:44:19,869 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/settings +2024-12-31 23:44:19,870 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 23:44:19,870 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 23:44:19,870 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 23:44:19,870 INFO MainThread:1769082 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 23:44:19,871 INFO MainThread:1769082 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/logs/debug.log +2024-12-31 23:44:19,871 INFO MainThread:1769082 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/logs/debug-internal.log +2024-12-31 23:44:19,871 INFO MainThread:1769082 [wandb_init.py:init():569] calling init triggers +2024-12-31 23:44:19,872 INFO MainThread:1769082 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf'} +2024-12-31 23:44:19,872 INFO MainThread:1769082 [wandb_init.py:init():619] starting backend +2024-12-31 23:44:19,872 INFO MainThread:1769082 [wandb_init.py:init():623] setting up manager +2024-12-31 23:44:19,876 INFO MainThread:1769082 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 23:44:19,877 INFO MainThread:1769082 [wandb_init.py:init():631] backend started and connected +2024-12-31 23:44:19,889 INFO MainThread:1769082 [wandb_init.py:init():720] updated telemetry +2024-12-31 23:44:19,895 INFO MainThread:1769082 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 23:44:20,120 INFO MainThread:1769082 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 23:44:20,158 INFO MainThread:1769082 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 23:44:20,158 INFO MainThread:1769082 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 23:44:20,501 INFO MainThread:1769082 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 23:44:20,501 INFO MainThread:1769082 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 23:44:20,501 INFO MainThread:1769082 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 23:44:20,501 INFO MainThread:1769082 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 23:44:20,502 INFO MainThread:1769082 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 23:44:20,502 INFO MainThread:1769082 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/fkfuijgf/tb, True +2024-12-31 23:44:24,662 INFO MainThread:1769082 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2024-12-31 23:52:13,952 WARNING MsgRouterThr:1769082 [router.py:message_loop():77] message_loop has been closed diff --git a/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/run-fkfuijgf.wandb b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/run-fkfuijgf.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a3c12527e2e8ff92368789e2e11a5c48e8df4932 Binary files /dev/null and b/fkfuijgf/wandb/run-20241231_234419-fkfuijgf/run-fkfuijgf.wandb differ diff --git a/lto4ps4o/.hydra/config.yaml b/lto4ps4o/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57f0faa93e8c5310ed098ffd123244267c23fcf8 --- /dev/null +++ b/lto4ps4o/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1745_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/lto4ps4o/.hydra/hydra.yaml b/lto4ps4o/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab3485f569e4802744b431889d74b7358606d279 --- /dev/null +++ b/lto4ps4o/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/lto4ps4o + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/lto4ps4o/.hydra/overrides.yaml b/lto4ps4o/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/lto4ps4o/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/lto4ps4o/eval_policy.log b/lto4ps4o/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..39f212ce7807cd402ec05e61546d16a7e17a17ef --- /dev/null +++ b/lto4ps4o/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-31 21:51:05,702][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 21:51:05,705][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-31 21:51:10,608][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/szs9rm8b/episode_rosbags/cam_tf_world.npy b/szs9rm8b/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/szs9rm8b/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/szs9rm8b/episode_rosbags/depth_K.npy b/szs9rm8b/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/szs9rm8b/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/szs9rm8b/eval_robot.log b/szs9rm8b/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..13f38cffbb5fba0f7bfa2e7c460425e048280adf --- /dev/null +++ b/szs9rm8b/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 13:15:56,863][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 13:15:56,866][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/uh3y27hm/episode_rosbags/episode_3_2025-01-01-13-33-05.bag b/uh3y27hm/episode_rosbags/episode_3_2025-01-01-13-33-05.bag new file mode 100644 index 0000000000000000000000000000000000000000..de140507612f4c03294970b2c97ede9c66e9c50f --- /dev/null +++ b/uh3y27hm/episode_rosbags/episode_3_2025-01-01-13-33-05.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f134430d0078bbd25fe8541cb575a95f116f35e0db16cd36bf35317b50d220f3 +size 2070692558