diff --git a/.gitattributes b/.gitattributes index 6aa0fb0bf3f74411c27cf24664914eb2ed349de9..12b693db79ad51b0de3364b33c6b28e1438d6a99 100644 --- a/.gitattributes +++ b/.gitattributes @@ -153,3 +153,43 @@ d4l0r92a/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text 2uvisx3r/episode_rosbags/episode_4_2025-01-07-20-38-14.bag filter=lfs diff=lfs merge=lfs -text jwgj4azt/episode_rosbags/episode_4_2025-01-06-20-34-40.bag filter=lfs diff=lfs merge=lfs -text i3pbiz9p/episode_rosbags/episode_1_2025-01-06-17-34-41.bag filter=lfs diff=lfs merge=lfs -text +1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 filter=lfs diff=lfs merge=lfs -text +1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 filter=lfs diff=lfs merge=lfs -text +i3pbiz9p/episode_rosbags/episode_3_2025-01-06-17-36-38.bag filter=lfs diff=lfs merge=lfs -text +i3pbiz9p/episode_rosbags/episode_2_2025-01-06-17-35-57.bag filter=lfs diff=lfs merge=lfs -text +1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 filter=lfs diff=lfs merge=lfs -text +1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 filter=lfs diff=lfs merge=lfs -text +1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 filter=lfs diff=lfs merge=lfs -text +2zm0rh5c/episode_rosbags/episode_2_2025-01-07-20-41-48.bag filter=lfs diff=lfs merge=lfs -text +1exa8l2r/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1exa8l2r/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1exa8l2r/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1exa8l2r/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1exa8l2r/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +d4l0r92a/episode_rosbags/episode_3_2025-01-06-17-31-41.bag filter=lfs diff=lfs merge=lfs -text +d4l0r92a/episode_rosbags/episode_1_2025-01-06-17-30-05.bag filter=lfs diff=lfs merge=lfs -text +9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 filter=lfs diff=lfs merge=lfs -text +i3pbiz9p/episode_rosbags/episode_0_2025-01-06-17-34-00.bag filter=lfs diff=lfs merge=lfs -text +9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 filter=lfs diff=lfs merge=lfs -text +9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 filter=lfs diff=lfs merge=lfs -text +9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 filter=lfs diff=lfs merge=lfs -text +9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 filter=lfs diff=lfs merge=lfs -text +9hy9qnsj/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +9hy9qnsj/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +9hy9qnsj/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +9hy9qnsj/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +9hy9qnsj/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +d4l0r92a/episode_rosbags/episode_4_2025-01-06-17-32-27.bag filter=lfs diff=lfs merge=lfs -text +d4l0r92a/episode_rosbags/episode_2_2025-01-06-17-30-57.bag filter=lfs diff=lfs merge=lfs -text +i3pbiz9p/episode_rosbags/episode_4_2025-01-06-17-37-35.bag filter=lfs diff=lfs merge=lfs -text +or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 filter=lfs diff=lfs merge=lfs -text +or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 filter=lfs diff=lfs merge=lfs -text +or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 filter=lfs diff=lfs merge=lfs -text +or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 filter=lfs diff=lfs merge=lfs -text +or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 filter=lfs diff=lfs merge=lfs -text +1exa8l2r/episode_rosbags/episode_1_2025-01-07-17-00-15.bag filter=lfs diff=lfs merge=lfs -text +or3sumxm/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +or3sumxm/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +or3sumxm/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +or3sumxm/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +or3sumxm/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/1exa8l2r/.hydra/config.yaml b/1exa8l2r/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/1exa8l2r/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/1exa8l2r/.hydra/hydra.yaml b/1exa8l2r/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3897a92efc1f3b4bffb1379661cfdf356d3c59a3 --- /dev/null +++ b/1exa8l2r/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/1exa8l2r/.hydra/overrides.yaml b/1exa8l2r/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/1exa8l2r/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/1exa8l2r/episode_rosbags/episode_1_2025-01-07-17-00-15.bag b/1exa8l2r/episode_rosbags/episode_1_2025-01-07-17-00-15.bag new file mode 100644 index 0000000000000000000000000000000000000000..f18674b7f95b663405c9c890df518499d1ad17c5 --- /dev/null +++ b/1exa8l2r/episode_rosbags/episode_1_2025-01-07-17-00-15.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48a1601d6adf5123543199954114c0ef184051831ed2962aa62cf1c04ed85f55 +size 1414139416 diff --git a/1exa8l2r/eval_video/0_eval.mp4 b/1exa8l2r/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8adc1ecc9d62ed183ba4eef02250b8cfee401280 --- /dev/null +++ b/1exa8l2r/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:133b41ccfd01da169e6f997a0afa86da57ca75bc95350675547df4da16c72247 +size 1199672 diff --git a/1exa8l2r/eval_video/1_eval.mp4 b/1exa8l2r/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..91bab13a602f3616ed624686f6fba5ed43998345 --- /dev/null +++ b/1exa8l2r/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46ad08891ecc2d017fe58026ff6e386d78bb5063c2fee9964b315d0063c69f8f +size 1174273 diff --git a/1exa8l2r/eval_video/2_eval.mp4 b/1exa8l2r/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..17da6315006c6f76354ed929d920bbe1251a9947 --- /dev/null +++ b/1exa8l2r/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b1402d0e72f1d58d0bdb670e9c7488146fc4e5bddeb42e2d6bb70f2d3af7a71 +size 1183879 diff --git a/1exa8l2r/eval_video/3_eval.mp4 b/1exa8l2r/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..614e98b58eb46403210331e8d8d832967e44476b --- /dev/null +++ b/1exa8l2r/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc74b0d17439e756067ec25794fd97de05f6106895cf968f3a0f99a96392774f +size 1190538 diff --git a/1exa8l2r/eval_video/4_eval.mp4 b/1exa8l2r/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..26f205d1906422bb2530d9b93239a55d29d6a0bf --- /dev/null +++ b/1exa8l2r/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e22e33fe5bb8bbb0d7c4824ea80bde87feb721157a0401ee63026422ed017f27 +size 1247290 diff --git a/1exa8l2r/tb/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 b/1exa8l2r/tb/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 new file mode 100644 index 0000000000000000000000000000000000000000..4b07b71baacafe4b789c47e2cd130199baaeeb45 --- /dev/null +++ b/1exa8l2r/tb/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2242bd861ab57c4707251b641f617d0ebaba452045d0617fc7ce103389117fc2 +size 6539 diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/code/FISH/eval_robot.py b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/config.yaml b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb9f1d627e7faf15b08b6ddb2859e3fb741f5982 --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1022_0' +wandb_notes: + desc: null + value: 1022_0_req_1348_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736287129 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/diff.patch b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a994978ad1dd8e2d18bdda54a765e73f2c0b5be --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/diff.patch @@ -0,0 +1,19 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..bc04fba 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -135,12 +135,12 @@ load_checkpoint: ${agent.load_checkpoint} + # RGBD+mask+act history + # wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 +-# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json new file mode 100644 index 0000000000000000000000000000000000000000..cc7e5e91447669fe989ab08b6e5b18934b0ea7ca --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.4, 0.04400911936343024, 0.8621948956718848, 0.05324718365067565, 0.803534984527916, 0.11712844677476308, 0.7487166672447589, 0.23282015328982517, 0.6637700199753331]]} \ No newline at end of file diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0f47b9199aa7777092bcb6d087f85736ef9cdce8 --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[15, 35, 0.42857142857142855, 0.2445112020288276, 0.6264930069193577, 0.27870236341610866, 0.6052635311869855, 0.30128256980578816, 0.56385952232991, 0.3175393715865753, 0.5477773211800865]]} \ No newline at end of file diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8adc1ecc9d62ed183ba4eef02250b8cfee401280 --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:133b41ccfd01da169e6f997a0afa86da57ca75bc95350675547df4da16c72247 +size 1199672 diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..91bab13a602f3616ed624686f6fba5ed43998345 --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46ad08891ecc2d017fe58026ff6e386d78bb5063c2fee9964b315d0063c69f8f +size 1174273 diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..17da6315006c6f76354ed929d920bbe1251a9947 --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b1402d0e72f1d58d0bdb670e9c7488146fc4e5bddeb42e2d6bb70f2d3af7a71 +size 1183879 diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..614e98b58eb46403210331e8d8d832967e44476b --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc74b0d17439e756067ec25794fd97de05f6106895cf968f3a0f99a96392774f +size 1190538 diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..26f205d1906422bb2530d9b93239a55d29d6a0bf --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e22e33fe5bb8bbb0d7c4824ea80bde87feb721157a0401ee63026422ed017f27 +size 1247290 diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c08d56053d610a959fac01f7f97e23b7154cd76a --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log @@ -0,0 +1,273 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736287137.825449]: resetting environment +[INFO] [1736287137.828837]: cleared current plan +[INFO] [1736287137.829678]: moving to home +[INFO] [1736287138.832026]: reached home +[INFO] [1736287138.833720]: reset action history +[INFO] [1736287141.546730]: environment reset +Starting episode 0 +[INFO] [1736287141.548968]: resetting environment +[INFO] [1736287141.551746]: cleared current plan +[INFO] [1736287141.553356]: moving to home +[INFO] [1736287142.555634]: reached home +[INFO] [1736287142.557499]: reset action history +[INFO] [1736287145.266210]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736287153.489039]: resetting environment +[INFO] [1736287153.497072]: cleared current plan +[INFO] [1736287153.497894]: moving to home +[INFO] [1736287154.500731]: reached home +[INFO] [1736287154.501980]: reset action history +[INFO] [1736287157.209747]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/episode_rosbags/episode_0_2025-01-07-16-59-17.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2192541.log +started roslaunch server http://158.130.50.37:33697/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2192829] +process[print_text-2]: started with pid [2192853] +process[rosbag_record-3]: started with pid [2192854] +started bagging! +For topic gripper_width: timestamp difference is 113725499 for nearest: 1736287166645520047 - target: 1736287166531794548 at idx 126 +For topic gripper_width: timestamp difference is 120760453 for nearest: 1736287175445522797 - target: 1736287175324762344 at idx 237 +[INFO] [1736287184.200444]: Storing episode... +[rosbag_record-3] killing on exit[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1736287184.929998]: Stored episode 1. +[INFO] [1736287184.930222]: Saving video... +[INFO] [1736287185.134696]: Video saved! +Starting episode 1 +[INFO] [1736287185.153864]: resetting environment +[INFO] [1736287185.160347]: cleared current plan +[INFO] [1736287185.160588]: moving to home +[INFO] [1736287188.368383]: reached home +[INFO] [1736287188.368826]: reset action history +[INFO] [1736287191.075192]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736287212.280875]: resetting environment +[INFO] [1736287212.284789]: cleared current plan +[INFO] [1736287212.285176]: moving to home +[INFO] [1736287213.286889]: reached home +[INFO] [1736287213.287755]: reset action history +[INFO] [1736287215.994133]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/episode_rosbags/episode_1_2025-01-07-17-00-15.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2192541.log +started roslaunch server http://158.130.50.37:34149/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2192968] +process[print_text-5]: started with pid [2192969] +process[rosbag_record-6]: started with pid [2192970] +started bagging! +For topic gripper_width: timestamp difference is 104153481 for nearest: 1736287232745522824 - target: 1736287232641369343 at idx 231 +For topic gripper_width: timestamp difference is 105220694 for nearest: 1736287235345565934 - target: 1736287235240345240 at idx 249 +For topic gripper_width: timestamp difference is 104068055 for nearest: 1736287235545565858 - target: 1736287235441497803 at idx 249 +For topic gripper_width: timestamp difference is 103811333 for nearest: 1736287236945477555 - target: 1736287236841666222 at idx 249 +For topic gripper_width: timestamp difference is 104049678 for nearest: 1736287237145554061 - target: 1736287237041504383 at idx 249 +For topic gripper_width: timestamp difference is 104503078 for nearest: 1736287237745522422 - target: 1736287237641019344 at idx 249 +For topic gripper_width: timestamp difference is 107224360 for nearest: 1736287241145520106 - target: 1736287241038295746 at idx 249 +[INFO] [1736287242.945840]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736287243.735485]: Stored episode 2. +[INFO] [1736287243.736022]: Saving video... +[WARN] [1736287244.297274]: Plan exhausted +[WARN] [1736287244.333432]: Plan exhausted +[WARN] [1736287244.375472]: Plan exhausted +[INFO] [1736287244.362292]: Video saved! +Starting episode 2 +[INFO] [1736287244.393886]: resetting environment +[INFO] [1736287244.400508]: cleared current plan +[INFO] [1736287244.400710]: moving to home +[INFO] [1736287247.705482]: reached home +[INFO] [1736287247.706522]: reset action history +[INFO] [1736287250.420451]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736287256.421148]: resetting environment +[INFO] [1736287256.432091]: cleared current plan +[INFO] [1736287256.433867]: moving to home +[INFO] [1736287257.436323]: reached home +[INFO] [1736287257.440420]: reset action history +[INFO] [1736287260.151633]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/episode_rosbags/episode_2_2025-01-07-17-01-00.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2192541.log +started roslaunch server http://158.130.50.37:42149/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2193105] +process[print_text-8]: started with pid [2193106] +process[rosbag_record-9]: started with pid [2193107] +started bagging! +For topic gripper_width: timestamp difference is 103840642 for nearest: 1736287278145520263 - target: 1736287278041679621 at idx 232 +[INFO] [1736287287.107518]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit[print_text-8] killing on exit +[INFO] [1736287287.888098]: Stored episode 3. +[INFO] [1736287287.889940]: Saving video... +[INFO] [1736287288.195432]: Video saved! +Starting episode 3 +[INFO] [1736287288.255987]: resetting environment +[INFO] [1736287288.269198]: cleared current plan +[INFO] [1736287288.271828]: moving to home +[INFO] [1736287291.579024]: reached home +[INFO] [1736287291.584895]: reset action history +[INFO] [1736287294.362453]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736287307.711478]: resetting environment +[INFO] [1736287307.730540]: cleared current plan +[INFO] [1736287307.733698]: moving to home +[INFO] [1736287308.737040]: reached home +[INFO] [1736287308.742301]: reset action history +[INFO] [1736287311.625125]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/episode_rosbags/episode_3_2025-01-07-17-01-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2192541.log +started roslaunch server http://158.130.50.37:43695/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2193240] +process[print_text-11]: started with pid [2193241] +process[rosbag_record-12]: started with pid [2193242] +started bagging! +[INFO] [1736287338.576699]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1736287339.403934]: Stored episode 4. +[INFO] [1736287339.407001]: Saving video... +[INFO] [1736287339.661472]: Video saved! +Starting episode 4 +[INFO] [1736287339.720482]: resetting environment +[INFO] [1736287339.731963]: cleared current plan +[INFO] [1736287339.735288]: moving to home +[INFO] [1736287343.241231]: reached home +[INFO] [1736287343.243029]: reset action history +[INFO] [1736287346.021013]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736287352.266095]: resetting environment +[INFO] [1736287352.296705]: cleared current plan +[INFO] [1736287352.300965]: moving to home +[INFO] [1736287353.305448]: reached home +[INFO] [1736287353.308974]: reset action history +[INFO] [1736287356.083938]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/episode_rosbags/episode_4_2025-01-07-17-02-36.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2192541.log +started roslaunch server http://158.130.50.37:46747/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2193400] +process[print_text-14]: started with pid [2193401] +process[rosbag_record-15]: started with pid [2193402] +started bagging! +[WARN] [1736287385.837389]: Plan exhausted +[WARN] [1736287385.876123]: Plan exhausted +[INFO] [1736287384.491136]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736287385.282177]: Stored episode 5. +[INFO] [1736287385.287178]: Saving video... +[INFO] [1736287385.820975]: Video saved! +[INFO] [1736287385.881773]: resetting environment +[INFO] [1736287385.909839]: cleared current plan +[INFO] [1736287385.916780]: moving to home +[INFO] [1736287389.433226]: reached home +[INFO] [1736287389.439920]: reset action history +[INFO] [1736287392.296066]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/requirements.txt b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-metadata.json b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5022e97497f9f136dc24d1c5f9194d361dc83fa2 --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T21:58:50.090144", + "startedAt": "2025-01-07T21:58:49.641407", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3934.3185000000003, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.953, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 723.8112297058105 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..eca9995cd9f900ce1d9d04937016d86c3669f545 --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "133b41ccfd01da169e6f997a0afa86da57ca75bc95350675547df4da16c72247", "size": 1199672, "path": "media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4"}, "global_step": 665, "_timestamp": 1736287414.7489116, "_runtime": 285.0930407047272, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "46ad08891ecc2d017fe58026ff6e386d78bb5063c2fee9964b315d0063c69f8f", "size": 1174273, "path": "media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4"}, "eval/num_success": 2.0, "episode": 35.0, "eval/success_rate": 0.4000000059604645, "eval/success_rate_lb_0.01": 0.044009119272232056, "eval/success_rate_ub_0.01": 0.8621948957443237, "eval/success_rate_lb_0.025": 0.05324718356132507, "eval/success_rate_ub_0.025": 0.803534984588623, "eval/success_rate_lb_0.05": 0.11712844669818878, "eval/success_rate_ub_0.05": 0.7487166523933411, "eval/success_rate_lb_0.1": 0.23282015323638916, "eval/success_rate_ub_0.1": 0.6637700200080872, "eval/2_eval": {"_type": "video-file", "sha256": "2b1402d0e72f1d58d0bdb670e9c7488146fc4e5bddeb42e2d6bb70f2d3af7a71", "size": 1183879, "path": "media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "dc74b0d17439e756067ec25794fd97de05f6106895cf968f3a0f99a96392774f", "size": 1190538, "path": "media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "e22e33fe5bb8bbb0d7c4824ea80bde87feb721157a0401ee63026422ed017f27", "size": 1247290, "path": "media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "85c54431b4eb7c8145a72a6437cbffccb84a9cf38051f18b07dd0a638d459f25", "size": 401, "artifact_path": "wandb-client-artifact://5yrnwuxfoostu66px7qumfmo44yimd7tnvicrwuf4c0lqj055rmp5s5uw7zs28vhikn17h9rl4h8ud9y6b6oorqke5dty773zgflcyd1cafjvgmbhfi4eqqbkc5yec7h/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://vil0hhlatci87uk0rnax4wyvp31lijanpfgz4svcbabexxvyfl1yrt7zl5sn3yq0thzc1uepglcfacdxurv9acd04caat2h1cxrct5nza4fto2anbh9y0xn53zkzgmg1:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 15, "total_num_episodes": 35, "total_success_rate": 0.42857142857142855, "total_success_rate_lb_0.01": 0.2445112020288276, "total_success_rate_ub_0.01": 0.6264930069193577, "total_success_rate_lb_0.025": 0.27870236341610866, "total_success_rate_ub_0.025": 0.6052635311869855, "total_success_rate_lb_0.05": 0.30128256980578816, "total_success_rate_ub_0.05": 0.56385952232991, "total_success_rate_lb_0.1": 0.3175393715865753, "total_success_rate_ub_0.1": 0.5477773211800865, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "a80255a63fbf0577db6185f15aab207c05191f208d1abc6b89f90822d076651e", "size": 521, "artifact_path": "wandb-client-artifact://1espmdchlw8vi8zq3hdcz2oy3apqx4d6bsi7yrlb2fvfso4ej8ihpn73rmw6tz0vjfv1yp8dzpzqpdlmrlkhymlq12hurgigwb41e10op4nurzfj72v894h8f0p3rs2z/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://bpm62zxaoohkekn15c5lbjd0c3yfcep257bz1cu0n90foj1buobw83l0d7cvcfoe9v5x3s4ofvkdf0ut420o4b0x82rashl2to5q3hiv5fm7xziqlhww5zc5qz58nck3:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 286}, "eval/total_success_rate": 0.4285714328289032, "eval/total_success_rate_lb_0.01": 0.2445112019777298, "eval/total_success_rate_ub_0.01": 0.626492977142334, "eval/total_success_rate_lb_0.025": 0.27870237827301025, "eval/total_success_rate_ub_0.025": 0.6052635312080383, "eval/total_success_rate_lb_0.05": 0.3012825846672058, "eval/total_success_rate_ub_0.05": 0.5638595223426819, "eval/total_success_rate_lb_0.1": 0.31753936409950256, "eval/total_success_rate_ub_0.1": 0.5477772951126099} \ No newline at end of file diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/logs/debug-internal.log b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..43777f567bf02b886650098dacd350052b3ab73f --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/logs/debug-internal.log @@ -0,0 +1,800 @@ +2025-01-07 16:58:49,655 INFO StreamThr :2192706 [internal.py:wandb_internal():85] W&B internal server running at pid: 2192706, started at: 2025-01-07 16:58:49.654614 +2025-01-07 16:58:49,655 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status +2025-01-07 16:58:49,668 INFO WriterThread:2192706 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/run-1exa8l2r.wandb +2025-01-07 16:58:49,668 DEBUG SenderThread:2192706 [sender.py:send():379] send: header +2025-01-07 16:58:49,675 DEBUG SenderThread:2192706 [sender.py:send():379] send: run +2025-01-07 16:58:49,962 INFO SenderThread:2192706 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files +2025-01-07 16:58:49,962 INFO SenderThread:2192706 [sender.py:_start_run_threads():1188] run started: 1exa8l2r with start time 1736287129.655871 +2025-01-07 16:58:49,978 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 16:58:49,979 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: check_version +2025-01-07 16:58:50,065 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 16:58:50,070 DEBUG HandlerThread:2192706 [system_info.py:__init__():26] System info init +2025-01-07 16:58:50,070 DEBUG HandlerThread:2192706 [system_info.py:__init__():41] System info init done +2025-01-07 16:58:50,070 INFO HandlerThread:2192706 [system_monitor.py:start():194] Starting system monitor +2025-01-07 16:58:50,070 INFO SystemMonitor:2192706 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 16:58:50,070 INFO HandlerThread:2192706 [system_monitor.py:probe():214] Collecting system info +2025-01-07 16:58:50,071 INFO SystemMonitor:2192706 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 16:58:50,071 INFO SystemMonitor:2192706 [interfaces.py:start():188] Started disk monitoring +2025-01-07 16:58:50,072 INFO SystemMonitor:2192706 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 16:58:50,072 INFO SystemMonitor:2192706 [interfaces.py:start():188] Started memory monitoring +2025-01-07 16:58:50,073 INFO SystemMonitor:2192706 [interfaces.py:start():188] Started network monitoring +2025-01-07 16:58:50,090 DEBUG HandlerThread:2192706 [system_info.py:probe():152] Probing system +2025-01-07 16:58:50,091 DEBUG HandlerThread:2192706 [system_info.py:_probe_git():137] Probing git +2025-01-07 16:58:50,096 DEBUG HandlerThread:2192706 [system_info.py:_probe_git():145] Probing git done +2025-01-07 16:58:50,096 DEBUG HandlerThread:2192706 [system_info.py:probe():200] Probing system done +2025-01-07 16:58:50,096 DEBUG HandlerThread:2192706 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T21:58:50.090144', 'startedAt': '2025-01-07T21:58:49.641407', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3934.3185000000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.953, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 723.8112297058105}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 16:58:50,096 INFO HandlerThread:2192706 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 16:58:50,096 INFO HandlerThread:2192706 [system_monitor.py:probe():227] Publishing system info +2025-01-07 16:58:50,096 DEBUG HandlerThread:2192706 [system_info.py:_save_code():44] Saving code +2025-01-07 16:58:50,102 DEBUG HandlerThread:2192706 [system_info.py:_save_code():67] Saving code done +2025-01-07 16:58:50,102 DEBUG HandlerThread:2192706 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 16:58:50,196 DEBUG HandlerThread:2192706 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 16:58:50,196 INFO HandlerThread:2192706 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 16:58:50,199 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 16:58:50,199 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 16:58:50,200 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 16:58:50,200 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 16:58:50,446 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/fx1h7118-wandb-metadata.json +2025-01-07 16:58:50,454 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 16:58:50,455 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: python_packages +2025-01-07 16:58:50,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:58:50,455 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:58:50,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:50,457 INFO HandlerThread:2192706 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/tb" + save: true +} +control { + relay_id: "8899518ba0484a498e6f13061035b9dd" +} +_info { + stream_id: "1exa8l2r" +} + +2025-01-07 16:58:50,457 DEBUG HandlerThread:2192706 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 16:58:50,472 INFO wandb-upload_1:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/m7e3sa56-code/FISH/eval_robot.py +2025-01-07 16:58:50,483 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/dthq6sum-diff.patch +2025-01-07 16:58:50,600 DEBUG SenderThread:2192706 [sender.py:send():379] send: telemetry +2025-01-07 16:58:50,600 DEBUG SenderThread:2192706 [sender.py:send():379] send: tbrecord +2025-01-07 16:58:50,600 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 16:58:50,600 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 with policy live +2025-01-07 16:58:50,780 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/xkk90gpj-events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 16:58:50,963 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:58:50,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/diff.patch +2025-01-07 16:58:50,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-metadata.json +2025-01-07 16:58:50,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/code/FISH/eval_robot.py +2025-01-07 16:58:50,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/requirements.txt +2025-01-07 16:58:50,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 16:58:50,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/code/FISH +2025-01-07 16:58:50,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/code +2025-01-07 16:58:51,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:52,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:52,963 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:58:53,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:54,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:55,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:58:55,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:56,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:57,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:57,825 DEBUG SenderThread:2192706 [sender.py:send():379] send: config +2025-01-07 16:58:58,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:58,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:58:59,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:00,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:00,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:00,965 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:01,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:02,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:02,966 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:03,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:04,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:04,966 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:05,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:59:05,455 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:59:05,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:05,611 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:06,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:06,967 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:07,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:08,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:09,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:10,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:11,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:11,458 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:12,458 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:13,459 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:14,459 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:14,969 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:15,459 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:16,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:16,459 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:16,970 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:17,461 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:18,461 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:18,970 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:19,462 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:20,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:59:20,455 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:59:20,497 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:21,462 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:21,592 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:21,971 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/config.yaml +2025-01-07 16:59:22,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:23,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:24,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:25,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:26,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:27,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:27,689 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:28,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:28,973 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:29,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:30,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:31,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:32,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:33,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:33,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:34,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:35,497 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:59:35,497 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:35,498 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:59:36,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:36,981 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:37,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:38,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:38,602 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:39,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:40,465 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:41,465 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:42,465 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:43,465 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:44,200 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:44,465 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:45,152 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 16:59:45,152 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 with policy now +2025-01-07 16:59:45,152 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:59:45,153 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 16:59:45,153 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:59:45,154 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:59:45,404 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/1byh3xhq-media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 +2025-01-07 16:59:45,465 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:45,984 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 +2025-01-07 16:59:45,984 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 16:59:45,984 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media +2025-01-07 16:59:45,984 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos +2025-01-07 16:59:45,985 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval +2025-01-07 16:59:46,260 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/oo76nf8c-media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 +2025-01-07 16:59:46,465 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:46,984 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:47,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:48,466 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:49,409 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:49,466 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:50,073 DEBUG SystemMonitor:2192706 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 16:59:50,074 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 16:59:50,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:59:50,455 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:59:50,501 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:50,986 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:51,466 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:52,466 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:52,986 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:53,466 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:54,466 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:55,118 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:55,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:56,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:57,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:58,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:59,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:00,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:00,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:01,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:02,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:03,468 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:04,468 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:05,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:00:05,455 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:00:05,497 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:05,523 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:06,468 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:07,468 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:08,468 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:09,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:10,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:11,458 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:11,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:12,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:12,993 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:00:13,237 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/dsn5nkcp-events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:00:13,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:14,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:14,994 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:15,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:16,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:16,994 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:17,222 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:17,470 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:18,470 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:18,995 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:19,470 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:20,075 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:00:20,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:00:20,456 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:00:20,497 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:21,470 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:22,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:22,574 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:23,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:24,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:25,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:26,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:27,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:28,458 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:28,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:29,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:30,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:31,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:32,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:33,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:33,797 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:34,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:35,000 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:35,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:00:35,455 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:00:35,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:36,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:37,001 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:37,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:38,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:39,002 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:39,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:39,797 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:40,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:41,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:42,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:43,003 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:43,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:44,385 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 17:00:44,385 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 with policy now +2025-01-07 17:00:44,388 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:00:44,388 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:00:44,388 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:00:44,395 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:00:44,474 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:44,639 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/lz0rmtpf-media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 +2025-01-07 17:00:45,004 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:00:45,004 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:45,004 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 +2025-01-07 17:00:45,004 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval +2025-01-07 17:00:45,280 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/sg1y8jde-media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 +2025-01-07 17:00:45,401 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:45,476 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:46,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:47,004 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:47,482 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:48,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:49,005 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:49,482 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:50,075 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:00:50,421 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:50,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:00:50,469 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:00:50,480 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:51,480 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:52,480 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:53,007 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:53,480 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:54,480 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:55,459 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:55,480 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:56,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:56,890 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:00:56,890 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:00:56,890 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:00:57,008 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:00:57,009 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:00:57,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:58,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:59,009 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:59,482 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:00,462 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:00,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:01,010 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:01,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:02,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:03,011 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:03,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:04,482 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:05,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:01:05,456 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:01:05,497 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:05,581 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:06,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:07,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:08,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:09,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:10,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:11,462 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:11,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:12,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:13,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:14,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:15,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:16,485 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:17,462 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:17,485 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:18,485 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:19,016 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:19,485 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:20,076 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:01:20,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:01:20,456 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:01:20,501 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:21,485 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:22,486 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:22,543 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:23,486 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:24,486 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:25,487 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:26,486 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:27,488 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:27,889 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:28,243 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 17:01:28,243 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 with policy now +2025-01-07 17:01:28,248 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:01:28,249 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:01:28,249 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:01:28,249 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:01:28,488 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:28,499 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/0g8ccs9p-media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 +2025-01-07 17:01:29,019 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:01:29,019 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:29,019 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 +2025-01-07 17:01:29,020 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval +2025-01-07 17:01:29,323 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/fh1frayt-media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 +2025-01-07 17:01:29,491 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:30,489 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:31,489 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:32,492 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:33,020 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:33,493 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:33,625 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:34,489 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:35,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:01:35,457 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:01:35,501 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:36,494 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:37,022 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:37,496 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:38,500 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:39,495 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:39,534 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:40,499 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:41,499 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:42,496 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:43,499 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:44,500 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:45,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:45,500 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:46,499 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:47,501 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:47,747 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:01:47,747 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:01:47,747 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:01:48,025 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:01:48,026 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:01:48,285 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/j7murppu-events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:01:48,506 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:49,026 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:49,505 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:50,076 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:01:50,458 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:01:50,458 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:01:50,506 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:50,537 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:51,027 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:51,612 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:52,599 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:53,027 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:53,598 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:54,598 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:55,598 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:56,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:56,598 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:57,599 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:58,598 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:59,599 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:00,599 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:01,599 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:02,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:02,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:03,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:04,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:05,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:02:05,457 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:02:05,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:06,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:07,594 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:07,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:08,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:09,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:10,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:11,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:12,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:13,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:13,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:14,601 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:15,601 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:16,601 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:17,601 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:18,577 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:18,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:19,604 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:19,713 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 17:02:19,713 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 with policy now +2025-01-07 17:02:19,716 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:02:19,716 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:02:19,716 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:02:19,717 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:02:19,972 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/bmos0zwk-media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 +2025-01-07 17:02:20,036 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:02:20,036 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 +2025-01-07 17:02:20,037 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval +2025-01-07 17:02:20,077 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:02:20,298 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/r6f9hx37-media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 +2025-01-07 17:02:20,461 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:02:20,461 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:02:20,609 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:21,037 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:02:21,609 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:22,613 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:23,607 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:24,285 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:24,610 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:25,038 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:02:25,612 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:26,611 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:27,039 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:02:27,613 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:28,612 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:29,612 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:30,066 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:30,616 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:31,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:32,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:32,800 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:02:32,801 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:02:32,802 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:02:33,041 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:02:33,041 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:02:33,041 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:02:33,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:34,621 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:35,041 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:02:35,349 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:35,466 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:02:35,466 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:02:35,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:36,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:37,042 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:02:37,615 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:38,615 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:39,043 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:02:39,615 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:40,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:40,615 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:41,616 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:42,616 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:43,616 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:44,616 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:45,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:45,616 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:46,616 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:47,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:48,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:49,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:50,077 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:02:50,458 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:02:50,458 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:02:50,593 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:50,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:51,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:52,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:53,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:54,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:55,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:56,475 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:56,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:57,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:58,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:59,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:00,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:01,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:02,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:02,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:03,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:04,619 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:05,468 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:03:05,468 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:03:05,628 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:05,879 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 17:03:05,879 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 with policy now +2025-01-07 17:03:05,880 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:03:05,880 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:03:05,880 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:05,881 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:06,062 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:03:06,062 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 +2025-01-07 17:03:06,062 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval +2025-01-07 17:03:06,138 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/6ccuuatk-media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 +2025-01-07 17:03:06,395 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/ii264243-media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 +2025-01-07 17:03:06,632 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:07,062 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:07,637 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:07,917 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:08,636 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:09,636 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:10,636 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:11,063 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:11,639 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:12,638 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:13,064 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:13,341 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:13,637 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:14,643 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:15,648 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:16,645 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:17,638 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:18,485 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:18,655 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:19,653 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:20,082 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:03:20,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:03:20,463 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:03:20,658 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:21,650 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:22,655 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:23,635 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:23,661 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:24,661 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:25,653 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:26,660 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:27,656 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:28,658 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:29,495 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:29,661 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:30,657 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:31,661 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:32,674 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:33,669 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:33,873 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:03:33,874 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:33,874 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:34,071 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:03:34,071 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:03:34,284 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:03:34,284 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:03:34,322 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/qnep3tbh-events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:03:34,462 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:03:34,665 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:34,779 INFO wandb-upload_0:2192706 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdfbhba_e +2025-01-07 17:03:34,845 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:03:34,853 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:03:35,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:03:35,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:35,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json +2025-01-07 17:03:35,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 17:03:35,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media +2025-01-07 17:03:35,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table +2025-01-07 17:03:35,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval +2025-01-07 17:03:35,243 INFO SenderThread:2192706 [sender.py:send_request_log_artifact():1525] logged artifact run-1exa8l2r-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDQ4Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY4MzE2Nw==', 'latestArtifact': None}} +2025-01-07 17:03:35,243 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 17:03:35,243 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json with policy now +2025-01-07 17:03:35,243 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:35,243 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:03:35,244 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,244 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,244 DEBUG SenderThread:2192706 [sender.py:send():379] send: telemetry +2025-01-07 17:03:35,244 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,248 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,248 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,249 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,249 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,249 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,249 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,250 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,250 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,250 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,251 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,251 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,251 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,251 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,252 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,252 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,252 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,252 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,252 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,253 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,253 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,253 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,253 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:03:35,442 INFO wandb-upload_1:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/rnwxrymp-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 17:03:35,453 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/pw46t55s-media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json +2025-01-07 17:03:35,460 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:03:35,665 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:35,689 INFO wandb-upload_1:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/idgarda9-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 17:03:35,795 INFO wandb-upload_0:2192706 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_j60bioa +2025-01-07 17:03:36,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:03:36,181 INFO SenderThread:2192706 [sender.py:send_request_log_artifact():1525] logged artifact run-1exa8l2r-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDUxNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY4MzE2OQ==', 'latestArtifact': None}} +2025-01-07 17:03:36,182 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 17:03:36,182 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json with policy now +2025-01-07 17:03:36,182 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:03:36,183 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:36,183 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:36,183 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:03:36,427 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/u9ans44y-media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json +2025-01-07 17:03:36,537 DEBUG SenderThread:2192706 [sender.py:send():379] send: exit +2025-01-07 17:03:36,537 INFO SenderThread:2192706 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:03:36,538 INFO SenderThread:2192706 [sender.py:send_exit():588] handling runtime: 286 +2025-01-07 17:03:36,538 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:36,538 INFO SenderThread:2192706 [sender.py:send_exit():594] send defer +2025-01-07 17:03:36,538 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:36,538 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:03:36,538 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:36,538 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:03:36,539 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:03:36,539 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:36,539 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:03:36,539 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:36,539 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:03:36,539 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:03:36,539 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:36,539 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:03:36,539 INFO HandlerThread:2192706 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:03:36,539 DEBUG SystemMonitor:2192706 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:03:36,539 DEBUG SystemMonitor:2192706 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:03:36,540 INFO HandlerThread:2192706 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:03:36,540 INFO HandlerThread:2192706 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:03:36,593 INFO HandlerThread:2192706 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:03:36,593 INFO HandlerThread:2192706 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:03:36,593 INFO HandlerThread:2192706 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:03:36,594 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:36,594 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:03:36,594 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:03:36,594 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:03:36,595 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:36,595 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:03:36,595 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:36,595 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:03:36,595 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:03:36,595 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:36,595 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:03:37,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:03:37,073 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:39,073 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:42,880 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:03:42,881 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:42,881 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:03:42,881 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:42,882 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:03:42,882 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:03:42,882 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:03:42,882 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:42,883 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:03:42,883 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:03:42,883 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:42,885 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:42,885 DEBUG SenderThread:2192706 [sender.py:send():379] send: summary +2025-01-07 17:03:42,885 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:42,885 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:42,885 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:03:42,885 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:03:42,885 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:42,886 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:03:42,886 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:42,886 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:03:42,889 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:43,012 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:03:43,013 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:43,013 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:03:43,013 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:43,013 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:03:43,075 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:03:43,075 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/config.yaml +2025-01-07 17:03:43,539 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:03:46,471 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:03:46,471 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:03:46,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:46,472 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:03:46,472 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:46,472 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:03:46,472 INFO SenderThread:2192706 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:03:46,474 INFO SenderThread:2192706 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:03:46,476 INFO SenderThread:2192706 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:03:46,490 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:03:46,490 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:46,490 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:03:46,490 DEBUG SenderThread:2192706 [sender.py:send():379] send: artifact +2025-01-07 17:03:46,539 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:03:46,995 INFO wandb-upload_1:2192706 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpn5jjzskl +2025-01-07 17:03:46,995 INFO wandb-upload_2:2192706 [upload_job.py:push():85] Skipped uploading /tmp/tmpfj24jvix/wandb-job.json +2025-01-07 17:03:47,077 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:47,175 INFO wandb-upload_0:2192706 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpr5j4j1ba +2025-01-07 17:03:47,710 INFO SenderThread:2192706 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTgyOTA5NQ==', 'versionIndex': 433}}} +2025-01-07 17:03:47,710 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:47,710 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:03:47,710 INFO SenderThread:2192706 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:03:48,077 INFO SenderThread:2192706 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files +2025-01-07 17:03:48,077 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/config.yaml config.yaml +2025-01-07 17:03:48,078 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json wandb-summary.json +2025-01-07 17:03:48,078 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:03:48,078 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/requirements.txt requirements.txt +2025-01-07 17:03:48,081 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:03:48,081 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log output.log +2025-01-07 17:03:48,081 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/diff.patch diff.patch +2025-01-07 17:03:48,081 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:03:48,082 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 +2025-01-07 17:03:48,082 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 +2025-01-07 17:03:48,082 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 +2025-01-07 17:03:48,082 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 +2025-01-07 17:03:48,082 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 +2025-01-07 17:03:48,084 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 17:03:48,084 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json +2025-01-07 17:03:48,084 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:03:48,084 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:03:48,084 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:48,084 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:03:48,087 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:48,087 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:03:48,087 INFO SenderThread:2192706 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:03:48,268 INFO wandb-upload_1:2192706 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/config.yaml +2025-01-07 17:03:48,276 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:03:48,289 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/requirements.txt +2025-01-07 17:03:48,292 INFO wandb-upload_4:2192706 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:48,342 INFO wandb-upload_3:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/yh7a92kv-events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:03:48,540 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:03:48,541 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:03:48,542 INFO Thread-11 (_thread_body):2192706 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:03:48,542 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:48,542 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:03:48,543 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:48,543 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:03:48,543 INFO SenderThread:2192706 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:03:48,544 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:03:48,544 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:48,544 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:03:48,544 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:48,544 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:03:48,544 INFO SenderThread:2192706 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:03:48,711 INFO SenderThread:2192706 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:03:48,711 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:03:48,711 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:48,711 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:03:48,712 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:48,712 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:03:48,712 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:03:48,712 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:48,712 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:03:48,712 DEBUG SenderThread:2192706 [sender.py:send():379] send: final +2025-01-07 17:03:48,712 DEBUG SenderThread:2192706 [sender.py:send():379] send: footer +2025-01-07 17:03:48,712 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:48,712 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:03:48,713 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:03:48,713 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:03:48,713 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:03:48,713 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:03:48,713 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:03:48,715 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:03:48,715 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:03:48,716 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:48,769 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:03:48,769 INFO MainThread:2192706 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:03:48,770 INFO MainThread:2192706 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:03:48,770 INFO MainThread:2192706 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:03:48,770 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:03:48,770 INFO HandlerThread:2192706 [handler.py:finish():882] shutting down handler +2025-01-07 17:03:49,713 INFO WriterThread:2192706 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/run-1exa8l2r.wandb +2025-01-07 17:03:49,769 INFO SenderThread:2192706 [sender.py:finish():1615] shutting down sender +2025-01-07 17:03:49,769 INFO SenderThread:2192706 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:03:49,769 INFO SenderThread:2192706 [file_pusher.py:join():175] waiting for file pusher diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/logs/debug.log b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2825e6008a3b55f92981e89475abb7e504d97e92 --- /dev/null +++ b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 16:58:49,645 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Configure stats pid to 2192541 +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/settings +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/logs/debug.log +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/logs/debug-internal.log +2025-01-07 16:58:49,647 INFO MainThread:2192541 [wandb_init.py:init():569] calling init triggers +2025-01-07 16:58:49,647 INFO MainThread:2192541 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r'} +2025-01-07 16:58:49,647 INFO MainThread:2192541 [wandb_init.py:init():619] starting backend +2025-01-07 16:58:49,648 INFO MainThread:2192541 [wandb_init.py:init():623] setting up manager +2025-01-07 16:58:49,653 INFO MainThread:2192541 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 16:58:49,655 INFO MainThread:2192541 [wandb_init.py:init():631] backend started and connected +2025-01-07 16:58:49,667 INFO MainThread:2192541 [wandb_init.py:init():720] updated telemetry +2025-01-07 16:58:49,673 INFO MainThread:2192541 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 16:58:49,978 INFO MainThread:2192541 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 16:58:50,016 INFO MainThread:2192541 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 16:58:50,017 INFO MainThread:2192541 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 16:58:50,455 INFO MainThread:2192541 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 16:58:50,455 INFO MainThread:2192541 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 16:58:50,455 INFO MainThread:2192541 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 16:58:50,455 INFO MainThread:2192541 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 16:58:50,457 INFO MainThread:2192541 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 16:58:50,457 INFO MainThread:2192541 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/tb, True +2025-01-07 16:58:57,824 INFO MainThread:2192541 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 17:03:49,771 WARNING MsgRouterThr:2192541 [router.py:message_loop():77] message_loop has been closed diff --git a/2zm0rh5c/episode_rosbags/episode_2_2025-01-07-20-41-48.bag b/2zm0rh5c/episode_rosbags/episode_2_2025-01-07-20-41-48.bag new file mode 100644 index 0000000000000000000000000000000000000000..8e11ca1b04aa4112bff2370ed82b054bffae8f71 --- /dev/null +++ b/2zm0rh5c/episode_rosbags/episode_2_2025-01-07-20-41-48.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3cec1f52c4515c9a7be2959473bcf87ec300824e36b00199412557dd0730e53 +size 1411285729 diff --git a/9hy9qnsj/.hydra/config.yaml b/9hy9qnsj/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/9hy9qnsj/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/9hy9qnsj/.hydra/hydra.yaml b/9hy9qnsj/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff1fc78c42ed26e8b5599f61283ff9ec3b31b966 --- /dev/null +++ b/9hy9qnsj/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/9hy9qnsj/.hydra/overrides.yaml b/9hy9qnsj/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/9hy9qnsj/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/9hy9qnsj/episode_rosbags/aligned_depth_to_color_K.npy b/9hy9qnsj/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/9hy9qnsj/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/9hy9qnsj/episode_rosbags/cam_tf_world.npy b/9hy9qnsj/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/9hy9qnsj/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/9hy9qnsj/episode_rosbags/color_K.npy b/9hy9qnsj/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/9hy9qnsj/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/9hy9qnsj/episode_rosbags/depth_K.npy b/9hy9qnsj/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/9hy9qnsj/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/9hy9qnsj/eval_robot.log b/9hy9qnsj/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..f1df432c09757ff056f186faa987e467884c0ad2 --- /dev/null +++ b/9hy9qnsj/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 17:04:09,933][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 17:04:09,936][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/9hy9qnsj/eval_video/0_eval.mp4 b/9hy9qnsj/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c3decb23a5bc63de1d9274d3b271e154f8830a01 --- /dev/null +++ b/9hy9qnsj/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3304018f8dfdbea23d5cb2d60ec083e3cf80c31d48b1c1430d16ce24e7ec1a7f +size 1187429 diff --git a/9hy9qnsj/eval_video/1_eval.mp4 b/9hy9qnsj/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0ad2b02e38661a44fd7d576ff7d20bd87244350b --- /dev/null +++ b/9hy9qnsj/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c9d0d50c4ae778950814daba6500046c4af2d18ae5b36b260f9c40191e6af53 +size 1202957 diff --git a/9hy9qnsj/eval_video/2_eval.mp4 b/9hy9qnsj/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..816f68e63e1b7bed1cfbe68cc0a6c208f72713d7 --- /dev/null +++ b/9hy9qnsj/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21ea113d4c8065f9636b9e31087621f164c18671087ecdbe14b870cfb260a9f2 +size 1194495 diff --git a/9hy9qnsj/eval_video/3_eval.mp4 b/9hy9qnsj/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8b51a740325914e2cb7d363b8bef574eeb1dec05 --- /dev/null +++ b/9hy9qnsj/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f865df86cb2d9b9e846969429ecae1975ae3c81cb840ed77e12ec9f89a066cd +size 1190183 diff --git a/9hy9qnsj/eval_video/4_eval.mp4 b/9hy9qnsj/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..572b95836d2405af76fcdae06efe33d269a5c881 --- /dev/null +++ b/9hy9qnsj/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f56992d8041cb7cbbc74437b2ef2daaf2e66f457a17ee0cbd9d5f504a2831b8b +size 1252175 diff --git a/9hy9qnsj/tb/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 b/9hy9qnsj/tb/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 new file mode 100644 index 0000000000000000000000000000000000000000..6a9ad8f54447e94c28e9b4b27240d2342acf8d1b --- /dev/null +++ b/9hy9qnsj/tb/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50909d8497dfc9998f44e3387dfeb40aae6ed7def04a8ac94ce73fe98c965b2d +size 6539 diff --git a/9hy9qnsj/wandb/debug-internal.log b/9hy9qnsj/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5f096e6cfe91ddf6e6e9cd5c00ff35d6035ad080 --- /dev/null +++ b/9hy9qnsj/wandb/debug-internal.log @@ -0,0 +1,767 @@ +2025-01-07 17:04:15,935 INFO StreamThr :2193794 [internal.py:wandb_internal():85] W&B internal server running at pid: 2193794, started at: 2025-01-07 17:04:15.935397 +2025-01-07 17:04:15,936 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:04:15,947 INFO WriterThread:2193794 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/run-9hy9qnsj.wandb +2025-01-07 17:04:15,948 DEBUG SenderThread:2193794 [sender.py:send():379] send: header +2025-01-07 17:04:15,959 DEBUG SenderThread:2193794 [sender.py:send():379] send: run +2025-01-07 17:04:16,202 INFO SenderThread:2193794 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files +2025-01-07 17:04:16,203 INFO SenderThread:2193794 [sender.py:_start_run_threads():1188] run started: 9hy9qnsj with start time 1736287455.937062 +2025-01-07 17:04:16,220 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:04:16,220 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:04:16,293 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:04:16,298 DEBUG HandlerThread:2193794 [system_info.py:__init__():26] System info init +2025-01-07 17:04:16,298 DEBUG HandlerThread:2193794 [system_info.py:__init__():41] System info init done +2025-01-07 17:04:16,298 INFO HandlerThread:2193794 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:04:16,298 INFO SystemMonitor:2193794 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:04:16,298 INFO HandlerThread:2193794 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:04:16,299 INFO SystemMonitor:2193794 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:04:16,300 INFO SystemMonitor:2193794 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:04:16,300 INFO SystemMonitor:2193794 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:04:16,301 INFO SystemMonitor:2193794 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:04:16,301 INFO SystemMonitor:2193794 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:04:16,311 DEBUG HandlerThread:2193794 [system_info.py:probe():152] Probing system +2025-01-07 17:04:16,312 DEBUG HandlerThread:2193794 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:04:16,317 DEBUG HandlerThread:2193794 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:04:16,317 DEBUG HandlerThread:2193794 [system_info.py:probe():200] Probing system done +2025-01-07 17:04:16,317 DEBUG HandlerThread:2193794 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:04:16.311164', 'startedAt': '2025-01-07T22:04:15.927913', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.31675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.575, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.433, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 730.4026374816895}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:04:16,317 INFO HandlerThread:2193794 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:04:16,317 INFO HandlerThread:2193794 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:04:16,317 DEBUG HandlerThread:2193794 [system_info.py:_save_code():44] Saving code +2025-01-07 17:04:16,319 DEBUG HandlerThread:2193794 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:04:16,319 DEBUG HandlerThread:2193794 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:04:16,383 DEBUG HandlerThread:2193794 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:04:16,384 INFO HandlerThread:2193794 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:04:16,387 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:04:16,387 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:04:16,387 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:04:16,387 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:04:16,622 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:04:16,622 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:04:16,622 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:04:16,623 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:04:16,624 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:16,641 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/my2aa3w4-wandb-metadata.json +2025-01-07 17:04:16,654 INFO wandb-upload_2:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/f7vytrbm-diff.patch +2025-01-07 17:04:16,658 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/ry4chh2f-code/FISH/eval_robot.py +2025-01-07 17:04:16,665 INFO HandlerThread:2193794 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/tb" + save: true +} +control { + relay_id: "7fa4d6ccfc7f47bdb9db499271e3d46c" +} +_info { + stream_id: "9hy9qnsj" +} + +2025-01-07 17:04:16,665 DEBUG HandlerThread:2193794 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:04:16,684 DEBUG SenderThread:2193794 [sender.py:send():379] send: telemetry +2025-01-07 17:04:16,716 DEBUG SenderThread:2193794 [sender.py:send():379] send: tbrecord +2025-01-07 17:04:16,717 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:04:16,717 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 with policy live +2025-01-07 17:04:16,913 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/ke0wobei-events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:04:17,204 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/diff.patch +2025-01-07 17:04:17,204 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:17,204 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:04:17,204 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/requirements.txt +2025-01-07 17:04:17,205 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/code/FISH/eval_robot.py +2025-01-07 17:04:17,205 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-metadata.json +2025-01-07 17:04:17,205 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/code/FISH +2025-01-07 17:04:17,205 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/code +2025-01-07 17:04:17,624 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:18,624 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:19,203 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:19,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:20,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:21,165 DEBUG SenderThread:2193794 [sender.py:send():379] send: config +2025-01-07 17:04:21,165 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:21,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:22,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:23,205 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:23,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:24,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:25,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:26,623 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:26,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:27,206 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:27,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:28,645 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:29,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:30,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:31,207 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:31,622 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:04:31,623 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:04:31,665 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:31,780 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:32,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:33,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:34,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:35,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:36,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:37,624 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:37,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:38,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:39,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:40,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:41,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:42,624 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:42,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:43,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:44,628 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:45,628 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:46,623 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:04:46,623 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:04:46,665 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:47,628 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:47,750 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:48,211 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/config.yaml +2025-01-07 17:04:48,628 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:49,629 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:50,628 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:51,212 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:51,629 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:52,629 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:53,213 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:53,526 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:53,630 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:54,630 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:55,214 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:55,630 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:56,630 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:57,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:58,624 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:58,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:59,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:00,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:01,623 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:05:01,623 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:05:01,665 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:02,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:03,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:03,776 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:04,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:05,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:06,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:07,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:08,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:09,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:09,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:10,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:11,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:12,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:13,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:14,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:14,909 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:15,219 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:15,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:16,301 DEBUG SystemMonitor:2193794 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:05:16,302 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:05:16,623 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:05:16,623 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:05:16,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:17,220 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:17,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:18,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:19,220 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:19,636 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:20,054 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:20,299 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:05:20,299 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 with policy now +2025-01-07 17:05:20,300 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:05:20,300 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:05:20,300 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:05:20,301 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:05:20,530 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/pm3lgumd-media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 +2025-01-07 17:05:20,636 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:21,221 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:21,221 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 +2025-01-07 17:05:21,221 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:05:21,221 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos +2025-01-07 17:05:21,222 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media +2025-01-07 17:05:21,222 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval +2025-01-07 17:05:21,501 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/80ci2ce8-media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 +2025-01-07 17:05:21,636 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:22,637 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:23,222 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:23,661 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:24,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:25,222 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:25,636 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:25,662 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:26,636 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:27,636 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:28,637 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:29,224 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:29,637 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:30,637 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:31,623 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:05:31,623 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:05:31,665 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:31,722 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:32,637 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:33,637 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:34,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:35,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:36,226 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:05:36,469 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/vh74ol48-events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:05:36,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:36,812 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:37,226 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:37,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:38,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:39,227 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:39,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:40,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:41,228 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:41,639 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:42,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:42,641 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:43,641 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:44,641 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:45,641 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:46,302 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:05:46,623 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:05:46,623 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:05:46,665 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:47,641 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:47,739 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:48,641 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:49,644 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:50,644 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:51,644 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:52,644 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:53,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:53,645 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:54,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:55,646 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:56,646 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:57,646 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:58,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:58,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:59,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:00,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:01,633 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:06:01,634 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:06:01,677 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:02,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:03,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:04,099 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:04,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:05,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:06,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:07,615 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:06:07,615 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 with policy now +2025-01-07 17:06:07,618 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:06:07,618 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:06:07,618 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:06:07,619 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:06:07,648 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:07,838 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/d3w0153m-media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 +2025-01-07 17:06:08,238 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:06:08,238 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 +2025-01-07 17:06:08,238 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval +2025-01-07 17:06:08,510 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/k9jm2vg4-media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 +2025-01-07 17:06:08,649 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:09,238 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:09,636 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:09,653 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:10,650 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:11,651 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:12,652 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:13,240 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:13,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:14,654 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:14,737 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:15,240 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:15,654 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:16,303 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:06:16,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:06:16,632 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:06:16,673 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:17,655 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:18,654 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:19,655 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:19,804 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:20,655 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:21,655 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:22,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:23,655 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:24,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:25,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:25,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:26,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:27,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:28,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:29,097 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:06:29,097 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:06:29,097 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:06:29,245 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:06:29,245 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:06:29,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:30,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:31,246 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:31,513 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:31,630 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:06:31,631 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:06:31,677 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:32,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:33,246 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:33,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:34,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:35,247 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:35,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:36,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:36,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:37,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:38,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:39,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:40,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:41,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:41,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:42,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:43,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:44,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:45,250 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:45,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:46,303 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:06:46,633 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:06:46,633 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:06:46,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:46,778 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:47,251 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:47,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:48,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:49,252 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:49,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:50,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:51,252 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:51,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:52,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:52,659 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:53,253 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:53,659 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:54,659 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:55,254 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:55,659 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:56,659 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:57,659 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:58,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:58,659 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:59,660 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:00,513 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:07:00,513 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 with policy now +2025-01-07 17:07:00,514 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:07:00,515 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:07:00,515 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:07:00,515 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:07:00,665 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:00,763 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/txi82k5l-media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 +2025-01-07 17:07:01,256 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:07:01,256 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:01,256 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 +2025-01-07 17:07:01,256 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval +2025-01-07 17:07:01,527 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/tq2ny0zk-media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 +2025-01-07 17:07:01,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:07:01,636 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:07:01,677 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:02,664 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:03,256 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:03,665 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:03,772 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:04,671 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:05,257 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:05,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:06,689 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:07,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:08,674 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:09,259 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:09,676 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:09,690 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:10,259 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:07:10,493 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/7xee8ci5-events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:07:10,681 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:10,935 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:07:10,935 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:07:10,936 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:07:11,259 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:07:11,260 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:11,675 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:12,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:13,260 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:13,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:14,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:15,105 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:15,261 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:15,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:16,304 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:07:16,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:07:16,632 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:07:16,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:17,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:18,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:19,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:20,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:20,738 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:21,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:22,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:23,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:24,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:25,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:26,634 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:26,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:27,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:28,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:29,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:30,685 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:31,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:07:31,632 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:07:31,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:31,790 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:32,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:33,267 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:33,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:34,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:35,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:36,681 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:37,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:37,681 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:38,681 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:39,681 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:40,681 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:41,725 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:42,007 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:07:42,008 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 with policy now +2025-01-07 17:07:42,018 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:07:42,019 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:07:42,019 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:07:42,020 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:07:42,260 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/re5ddgss-media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 +2025-01-07 17:07:42,270 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:07:42,270 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 +2025-01-07 17:07:42,270 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval +2025-01-07 17:07:42,527 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/njy744f5-media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 +2025-01-07 17:07:42,692 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:43,060 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:43,270 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:43,691 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:44,695 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:45,689 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:46,304 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:07:46,639 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:07:46,639 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:07:46,696 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:47,272 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:47,692 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:48,181 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:48,696 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:49,273 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:49,701 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:50,694 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:51,705 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:52,706 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:52,989 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:07:52,989 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:07:52,990 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:07:53,274 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:07:53,274 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:07:53,707 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:53,783 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:54,700 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:55,275 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:55,705 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:56,701 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:57,703 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:58,700 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:59,276 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:59,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:59,700 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:00,700 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:01,277 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:01,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:08:01,632 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:08:01,700 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:02,701 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:03,700 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:04,701 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:05,605 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:05,701 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:06,707 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:07,279 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:07,707 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:08,707 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:09,708 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:10,709 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:11,280 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:11,605 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:11,709 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:12,709 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:13,713 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:14,713 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:15,713 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:16,305 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:08:16,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:08:16,632 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:08:16,713 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:16,784 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:17,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:18,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:19,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:20,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:21,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:22,639 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:22,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:23,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:24,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:25,719 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:26,535 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:08:26,535 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 with policy now +2025-01-07 17:08:26,536 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:08:26,536 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:08:26,536 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:26,537 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:26,721 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:26,758 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/nso8q348-media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 +2025-01-07 17:08:27,286 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:08:27,286 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:27,286 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 +2025-01-07 17:08:27,286 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval +2025-01-07 17:08:27,586 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/yy3gk1va-media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 +2025-01-07 17:08:27,646 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:27,724 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:28,728 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:29,287 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:29,731 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:30,728 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:31,287 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:31,644 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:08:31,644 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:08:31,729 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:32,726 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:32,735 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:33,739 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:34,741 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:35,289 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:35,289 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:08:35,517 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/cm3kasi4-events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:08:35,731 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:36,043 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:08:36,043 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:36,043 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:36,160 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:08:36,160 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:08:36,289 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:08:36,289 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:08:36,289 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media +2025-01-07 17:08:36,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table +2025-01-07 17:08:36,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval +2025-01-07 17:08:36,304 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:08:36,460 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/vtqzykfj-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:08:36,682 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:08:36,691 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:08:36,728 INFO wandb-upload_1:2193794 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphq5633y_ +2025-01-07 17:08:36,731 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:37,183 INFO SenderThread:2193794 [sender.py:send_request_log_artifact():1525] logged artifact run-9hy9qnsj-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTg3OTkwOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY4NzU3NQ==', 'latestArtifact': None}} +2025-01-07 17:08:37,183 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:08:37,183 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 17:08:37,183 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:08:37,183 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,184 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,184 DEBUG SenderThread:2193794 [sender.py:send():379] send: telemetry +2025-01-07 17:08:37,184 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,188 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,188 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,189 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,189 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,189 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,189 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,190 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,190 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,190 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,190 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,191 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,191 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,191 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,191 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,191 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,192 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,192 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,192 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,192 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,192 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,193 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,193 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:08:37,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:08:37,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:37,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:08:37,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json +2025-01-07 17:08:37,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval +2025-01-07 17:08:37,427 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/yzjsydde-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:08:37,517 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/itcbvo4b-media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json +2025-01-07 17:08:37,532 INFO wandb-upload_2:2193794 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpio2exkdw +2025-01-07 17:08:37,731 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:37,947 INFO SenderThread:2193794 [sender.py:send_request_log_artifact():1525] logged artifact run-9hy9qnsj-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTg3OTk2Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY4NzU3OQ==', 'latestArtifact': None}} +2025-01-07 17:08:37,951 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:38,048 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:08:38,048 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json with policy now +2025-01-07 17:08:38,048 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:08:38,048 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:38,049 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:38,280 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/osijn9k6-media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json +2025-01-07 17:08:38,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:08:38,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/config.yaml +2025-01-07 17:08:38,369 DEBUG SenderThread:2193794 [sender.py:send():379] send: exit +2025-01-07 17:08:38,370 INFO SenderThread:2193794 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:08:38,370 INFO SenderThread:2193794 [sender.py:send_exit():588] handling runtime: 262 +2025-01-07 17:08:38,370 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:38,370 INFO SenderThread:2193794 [sender.py:send_exit():594] send defer +2025-01-07 17:08:38,370 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:38,370 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:08:38,370 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:38,371 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:08:38,371 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:08:38,371 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:38,371 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:08:38,371 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:38,371 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:08:38,371 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:08:38,371 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:38,371 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:08:38,371 INFO HandlerThread:2193794 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:08:38,371 DEBUG SystemMonitor:2193794 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:08:38,371 DEBUG SystemMonitor:2193794 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:08:38,372 INFO HandlerThread:2193794 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:08:38,372 INFO HandlerThread:2193794 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:08:38,425 INFO HandlerThread:2193794 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:08:38,425 INFO HandlerThread:2193794 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:08:38,425 INFO HandlerThread:2193794 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:08:38,425 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:38,425 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:08:38,426 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:08:38,426 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:08:38,426 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:38,426 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:08:38,426 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:38,426 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:08:38,426 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:08:38,426 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:38,426 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:08:39,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:08:39,291 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:45,049 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:08:45,049 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:45,049 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:08:45,050 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:45,050 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:08:45,050 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:08:45,051 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:45,051 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:08:45,051 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:08:45,051 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:08:45,051 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:45,052 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:45,052 DEBUG SenderThread:2193794 [sender.py:send():379] send: summary +2025-01-07 17:08:45,056 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:45,056 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:45,056 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:08:45,056 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:08:45,056 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:45,056 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:08:45,057 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:45,057 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:08:45,057 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:08:45,057 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:45,057 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:45,057 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:08:45,057 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:45,057 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:08:45,293 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:08:45,371 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:08:48,591 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:08:48,591 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:08:48,592 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:48,592 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:08:48,592 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:48,592 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:08:48,592 INFO SenderThread:2193794 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:08:48,594 INFO SenderThread:2193794 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:08:48,596 INFO SenderThread:2193794 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:08:48,607 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:08:48,607 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:48,607 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:08:48,607 DEBUG SenderThread:2193794 [sender.py:send():379] send: artifact +2025-01-07 17:08:49,091 INFO SenderThread:2193794 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'versionIndex': 434}}} +2025-01-07 17:08:49,091 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:49,091 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:08:49,091 INFO SenderThread:2193794 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:08:49,294 INFO SenderThread:2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:49,294 INFO SenderThread:2193794 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files +2025-01-07 17:08:49,294 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/config.yaml config.yaml +2025-01-07 17:08:49,294 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json wandb-summary.json +2025-01-07 17:08:49,294 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:08:49,295 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/requirements.txt requirements.txt +2025-01-07 17:08:49,295 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:08:49,295 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log output.log +2025-01-07 17:08:49,295 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/diff.patch diff.patch +2025-01-07 17:08:49,297 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:08:49,299 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 +2025-01-07 17:08:49,299 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 +2025-01-07 17:08:49,299 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 +2025-01-07 17:08:49,299 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 +2025-01-07 17:08:49,299 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 +2025-01-07 17:08:49,304 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:08:49,304 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json +2025-01-07 17:08:49,304 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:08:49,304 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:49,304 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:08:49,304 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:49,305 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:08:49,305 INFO SenderThread:2193794 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:08:49,372 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:08:49,372 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:08:49,467 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:08:49,486 INFO wandb-upload_2:2193794 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/config.yaml +2025-01-07 17:08:49,525 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/requirements.txt +2025-01-07 17:08:49,533 INFO wandb-upload_3:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/rhjt2y7r-events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:08:49,562 INFO wandb-upload_4:2193794 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:49,762 INFO Thread-11 (_thread_body):2193794 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:08:49,762 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:49,762 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:08:49,762 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:49,762 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:08:49,763 INFO SenderThread:2193794 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:08:49,764 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:08:49,764 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:49,764 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:08:49,764 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:49,764 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:08:49,764 INFO SenderThread:2193794 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:08:49,844 INFO SenderThread:2193794 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:08:49,845 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:08:49,845 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:49,845 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:08:49,845 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:49,845 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:08:49,845 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:08:49,845 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:49,845 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:08:49,845 DEBUG SenderThread:2193794 [sender.py:send():379] send: final +2025-01-07 17:08:49,846 DEBUG SenderThread:2193794 [sender.py:send():379] send: footer +2025-01-07 17:08:49,846 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:49,846 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:08:49,846 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:08:49,846 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:08:49,847 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:08:49,847 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:08:49,847 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:08:49,847 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:08:49,847 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:08:49,849 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:49,907 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:08:49,908 INFO MainThread:2193794 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:08:49,908 INFO MainThread:2193794 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:08:49,908 INFO MainThread:2193794 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:08:49,908 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:08:49,909 INFO HandlerThread:2193794 [handler.py:finish():882] shutting down handler +2025-01-07 17:08:50,847 INFO WriterThread:2193794 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/run-9hy9qnsj.wandb +2025-01-07 17:08:50,908 INFO SenderThread:2193794 [sender.py:finish():1615] shutting down sender +2025-01-07 17:08:50,908 INFO SenderThread:2193794 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:08:50,908 INFO SenderThread:2193794 [file_pusher.py:join():175] waiting for file pusher diff --git a/9hy9qnsj/wandb/debug.log b/9hy9qnsj/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..09ef84a31b46b5d43251a740170af934f6f08957 --- /dev/null +++ b/9hy9qnsj/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Configure stats pid to 2193627 +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/settings +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/logs/debug.log +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/logs/debug-internal.log +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj'} +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_init.py:init():619] starting backend +2025-01-07 17:04:15,931 INFO MainThread:2193627 [wandb_init.py:init():623] setting up manager +2025-01-07 17:04:15,934 INFO MainThread:2193627 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:04:15,936 INFO MainThread:2193627 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:04:15,947 INFO MainThread:2193627 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:04:15,956 INFO MainThread:2193627 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:04:16,219 INFO MainThread:2193627 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:04:16,286 INFO MainThread:2193627 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:04:16,286 INFO MainThread:2193627 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:04:16,624 INFO MainThread:2193627 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:04:16,624 INFO MainThread:2193627 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:04:16,624 INFO MainThread:2193627 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:04:16,624 INFO MainThread:2193627 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:04:16,626 INFO MainThread:2193627 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:04:16,626 INFO MainThread:2193627 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/tb, True +2025-01-07 17:04:21,164 INFO MainThread:2193627 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 17:08:50,909 WARNING MsgRouterThr:2193627 [router.py:message_loop():77] message_loop has been closed diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/code/FISH/eval_robot.py b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/config.yaml b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04e083de2e43df5cad822fd917598d308c192ff1 --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1022_0' +wandb_notes: + desc: null + value: 1022_0_req_1348_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736287455 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/diff.patch b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a994978ad1dd8e2d18bdda54a765e73f2c0b5be --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/diff.patch @@ -0,0 +1,19 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..bc04fba 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -135,12 +135,12 @@ load_checkpoint: ${agent.load_checkpoint} + # RGBD+mask+act history + # wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 +-# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json new file mode 100644 index 0000000000000000000000000000000000000000..bfa10a3bd141100c653477172dc1743b988d2eff --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[15, 40, 0.375, 0.21062204247540184, 0.561846166836767, 0.2405351996940744, 0.5408638417639066, 0.2605688684167502, 0.5018224492665461, 0.2753894329520266, 0.4858888499466987]]} \ No newline at end of file diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c3decb23a5bc63de1d9274d3b271e154f8830a01 --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3304018f8dfdbea23d5cb2d60ec083e3cf80c31d48b1c1430d16ce24e7ec1a7f +size 1187429 diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0ad2b02e38661a44fd7d576ff7d20bd87244350b --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c9d0d50c4ae778950814daba6500046c4af2d18ae5b36b260f9c40191e6af53 +size 1202957 diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..816f68e63e1b7bed1cfbe68cc0a6c208f72713d7 --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21ea113d4c8065f9636b9e31087621f164c18671087ecdbe14b870cfb260a9f2 +size 1194495 diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8b51a740325914e2cb7d363b8bef574eeb1dec05 --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f865df86cb2d9b9e846969429ecae1975ae3c81cb840ed77e12ec9f89a066cd +size 1190183 diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..572b95836d2405af76fcdae06efe33d269a5c881 --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f56992d8041cb7cbbc74437b2ef2daaf2e66f457a17ee0cbd9d5f504a2831b8b +size 1252175 diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..91f6028032500e7a9fb7201e9e4bb57d5185def9 --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log @@ -0,0 +1,285 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736287461.165128]: resetting environment +[INFO] [1736287461.168319]: cleared current plan +[INFO] [1736287461.169278]: moving to home +[INFO] [1736287462.170684]: reached home +[INFO] [1736287462.172199]: reset action history +[INFO] [1736287464.881432]: environment reset +Starting episode 0 +[INFO] [1736287464.883876]: resetting environment +[INFO] [1736287464.887195]: cleared current plan +[INFO] [1736287464.887961]: moving to home +[INFO] [1736287465.890779]: reached home +[INFO] [1736287465.892147]: reset action history +[INFO] [1736287468.599999]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736287488.508390]: resetting environment +[INFO] [1736287488.511711]: cleared current plan +[INFO] [1736287488.512560]: moving to home +[INFO] [1736287489.515253]: reached home +[INFO] [1736287489.516579]: reset action history +[INFO] [1736287492.224385]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/episode_rosbags/episode_0_2025-01-07-17-04-52.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2193627.log +started roslaunch server http://158.130.50.37:45171/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2193894] +process[print_text-2]: started with pid [2193918] +process[rosbag_record-3]: started with pid [2193919] +started bagging! +For topic gripper_width: timestamp difference is -105664569 for nearest: 1736287513678853911 - target: 1736287513784518480 at idx 249 +For topic gripper_width: timestamp difference is -116373856 for nearest: 1736287515878824632 - target: 1736287515995198488 at idx 249 +For topic gripper_width: timestamp difference is -115578151 for nearest: 1736287516278846764 - target: 1736287516394424915 at idx 249 +For topic gripper_width: timestamp difference is -114756121 for nearest: 1736287516678847538 - target: 1736287516793603659 at idx 249 +[INFO] [1736287519.240391]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736287520.053819]: Stored episode 1. +[INFO] [1736287520.054122]: Saving video... +[INFO] [1736287520.284300]: Video saved! +Starting episode 1 +[INFO] [1736287520.302116]: resetting environment +[INFO] [1736287520.311129]: cleared current plan +[INFO] [1736287520.314420]: moving to home +[INFO] [1736287523.618762]: reached home +[INFO] [1736287523.619336]: reset action history +[INFO] [1736287526.338454]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736287535.807335]: resetting environment +[INFO] [1736287535.809856]: cleared current plan +[INFO] [1736287535.810031]: moving to home +[INFO] [1736287536.811779]: reached home +[INFO] [1736287536.812184]: reset action history +[INFO] [1736287539.518782]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/episode_rosbags/episode_1_2025-01-07-17-05-39.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2193627.log +started roslaunch server http://158.130.50.37:42919/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2194031] +process[print_text-5]: started with pid [2194032] +process[rosbag_record-6]: started with pid [2194033] +started bagging! +[INFO] [1736287566.474263]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736287567.235981]: Stored episode 2. +[INFO] [1736287567.236581]: Saving video... +[INFO] [1736287567.594051]: Video saved! +Starting episode 2 +[INFO] [1736287567.623626]: resetting environment +[INFO] [1736287567.635134]: cleared current plan +[INFO] [1736287567.635798]: moving to home +[INFO] [1736287570.939283]: reached home +[INFO] [1736287570.948669]: reset action history +[INFO] [1736287573.690432]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736287588.458222]: resetting environment +[INFO] [1736287588.464417]: cleared current plan +[INFO] [1736287588.464720]: moving to home +[INFO] [1736287589.471721]: reached home +[INFO] [1736287589.472357]: reset action history +[INFO] [1736287592.181004]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/episode_rosbags/episode_2_2025-01-07-17-06-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2193627.log +started roslaunch server http://158.130.50.37:41681/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2194167] +process[print_text-8]: started with pid [2194168] +process[rosbag_record-9]: started with pid [2194169] +started bagging! +For topic gripper_width: timestamp difference is 100077920 for nearest: 1736287603545500093 - target: 1736287603445422173 at idx 141 +For topic gripper_width: timestamp difference is 100964838 for nearest: 1736287604545538479 - target: 1736287604444573641 at idx 153 +For topic gripper_width: timestamp difference is 101057448 for nearest: 1736287604745464244 - target: 1736287604644406796 at idx 155 +For topic gripper_width: timestamp difference is 102557627 for nearest: 1736287606545517651 - target: 1736287606442960024 at idx 180 +For topic gripper_width: timestamp difference is 102727332 for nearest: 1736287606745517411 - target: 1736287606642790079 at idx 182 +For topic gripper_width: timestamp difference is 103239832 for nearest: 1736287607345522223 - target: 1736287607242282391 at idx 188 +For topic gripper_width: timestamp difference is 104032698 for nearest: 1736287608345516863 - target: 1736287608241484165 at idx 197 +For topic gripper_width: timestamp difference is 104485689 for nearest: 1736287608945516525 - target: 1736287608841030836 at idx 204 +For topic gripper_width: timestamp difference is 104777308 for nearest: 1736287609145657511 - target: 1736287609040880203 at idx 206 +For topic gripper_width: timestamp difference is 105108500 for nearest: 1736287609745515109 - target: 1736287609640406609 at idx 215 +For topic gripper_width: timestamp difference is 105587849 for nearest: 1736287610345514664 - target: 1736287610239926815 at idx 224 +For topic gripper_width: timestamp difference is 105764815 for nearest: 1736287610545529792 - target: 1736287610439764977 at idx 226 +For topic gripper_width: timestamp difference is 108189701 for nearest: 1736287613545515417 - target: 1736287613437325716 at idx 249 +For topic gripper_width: timestamp difference is 108825940 for nearest: 1736287614345498818 - target: 1736287614236672878 at idx 249 +[INFO] [1736287619.145482]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736287619.920403]: Stored episode 3. +[INFO] [1736287619.922682]: Saving video... +[WARN] [1736287620.485613]: Plan exhausted +[WARN] [1736287620.525682]: Plan exhausted +[INFO] [1736287620.467996]: Video saved! +Starting episode 3 +[INFO] [1736287620.519881]: resetting environment +[INFO] [1736287620.529758]: cleared current plan +[INFO] [1736287620.531357]: moving to home +[INFO] [1736287623.842417]: reached home +[INFO] [1736287623.850889]: reset action history +[INFO] [1736287626.646511]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736287630.049753]: resetting environment +[INFO] [1736287630.062963]: cleared current plan +[INFO] [1736287630.063164]: moving to home +[INFO] [1736287631.064560]: reached home +[INFO] [1736287631.069433]: reset action history +[INFO] [1736287633.847222]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/episode_rosbags/episode_3_2025-01-07-17-07-13.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2193627.log +started roslaunch server http://158.130.50.37:44191/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2194304] +process[print_text-11]: started with pid [2194305] +process[rosbag_record-12]: started with pid [2194329] +started bagging! +For topic gripper_width: timestamp difference is 104885212 for nearest: 1736287650645516029 - target: 1736287650540630817 at idx 244 +[INFO] [1736287660.821229]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736287661.679686]: Stored episode 4. +[INFO] [1736287661.688203]: Saving video... +[INFO] [1736287661.945837]: Video saved! +Starting episode 4 +[INFO] [1736287662.027951]: resetting environment +[INFO] [1736287662.051142]: cleared current plan +[INFO] [1736287662.057892]: moving to home +[INFO] [1736287665.368970]: reached home +[INFO] [1736287665.384436]: reset action history +[INFO] [1736287668.181314]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736287672.749930]: resetting environment +[INFO] [1736287672.771113]: cleared current plan +[INFO] [1736287672.775845]: moving to home +[INFO] [1736287673.782588]: reached home +[INFO] [1736287673.784823]: reset action history +[INFO] [1736287676.578552]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/episode_rosbags/episode_4_2025-01-07-17-07-56.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2193627.log +started roslaunch server http://158.130.50.37:44671/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2194461] +process[print_text-14]: started with pid [2194462] +process[rosbag_record-15]: started with pid [2194463] +started bagging! +For topic gripper_width: timestamp difference is -202171267 for nearest: 1736287684278850137 - target: 1736287684481021404 at idx 103 +For topic gripper_width: timestamp difference is 105948270 for nearest: 1736287688545515121 - target: 1736287688439566851 at idx 148 +[INFO] [1736287705.334510]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736287706.152248]: Stored episode 5. +[INFO] [1736287706.161791]: Saving video... +[INFO] [1736287706.482574]: Video saved! +[INFO] [1736287706.537175]: resetting environment +[INFO] [1736287706.553530]: cleared current plan +[INFO] [1736287706.557239]: moving to home +[INFO] [1736287709.871070]: reached home +[INFO] [1736287709.887296]: reset action history +[INFO] [1736287712.744462]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/requirements.txt b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-metadata.json b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d06141fbafbb2ff13a54fc7dcf2402b0c33cfb7b --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T22:04:16.311164", + "startedAt": "2025-01-07T22:04:15.927913", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.31675, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.575, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.433, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 730.4026374816895 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8c7944afdd3979d1af09d112ae89f4e32e1b2ab8 --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "3304018f8dfdbea23d5cb2d60ec083e3cf80c31d48b1c1430d16ce24e7ec1a7f", "size": 1187429, "path": "media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4"}, "global_step": 665, "_timestamp": 1736287716.5980926, "_runtime": 260.66103053092957, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "1c9d0d50c4ae778950814daba6500046c4af2d18ae5b36b260f9c40191e6af53", "size": 1202957, "path": "media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4"}, "eval/num_success": 0.0, "episode": 40.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "21ea113d4c8065f9636b9e31087621f164c18671087ecdbe14b870cfb260a9f2", "size": 1194495, "path": "media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "7f865df86cb2d9b9e846969429ecae1975ae3c81cb840ed77e12ec9f89a066cd", "size": 1190183, "path": "media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "f56992d8041cb7cbbc74437b2ef2daaf2e66f457a17ee0cbd9d5f504a2831b8b", "size": 1252175, "path": "media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://8u0ovpff5dkb23e3qs9pxp1kcoy2584yvio75h2detwrr1wrqo633rtrpxnvblbbrr9o4us5rexs727u6xblypkk1t19ybzzlx0apvceo1u6cuy5p6mu58qwsjcld8hl/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://irlw1jzj5rkruy2dt5vhkw1qtz05gjapfwhjq7265p3pyzbv8topfs2l6fkt6546c8smxi4ox1na8l1xyp2i86cfp0k94svja0vzkm3m3kywoh2euxe78nirchmo5lq3:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 15, "total_num_episodes": 40, "total_success_rate": 0.375, "total_success_rate_lb_0.01": 0.21062204247540184, "total_success_rate_ub_0.01": 0.561846166836767, "total_success_rate_lb_0.025": 0.2405351996940744, "total_success_rate_ub_0.025": 0.5408638417639066, "total_success_rate_lb_0.05": 0.2605688684167502, "total_success_rate_ub_0.05": 0.5018224492665461, "total_success_rate_lb_0.1": 0.2753894329520266, "total_success_rate_ub_0.1": 0.4858888499466987, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "74d0bc1c5ec1209580e129ab1bd67db887d182981202e5ada86ace1ac9ce1861", "size": 507, "artifact_path": "wandb-client-artifact://5esq0udxt7r2q1kbg2ea7ly7z4arrddlg99gf3gwcw2mdj78vuhjfr00yu6mzyfjej2v0dy9vnv8dgzmz5jg5aquhnchvoexi4070cikd0n72jfthdyargww0pzb9kwk/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://xa69g142mblgjgo9tnweqb20boork2kotzbvy89i1vrms87xu75p8z9ijgbf8iicibikmcm8tkjdjl9o9pq0uupyqysmt72u24mcm5n8xp65wqqhz3ht75cy7f43u4tc:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 262}, "eval/total_success_rate": 0.375, "eval/total_success_rate_lb_0.01": 0.21062204241752625, "eval/total_success_rate_ub_0.01": 0.561846137046814, "eval/total_success_rate_lb_0.025": 0.2405351996421814, "eval/total_success_rate_ub_0.025": 0.5408638119697571, "eval/total_success_rate_lb_0.05": 0.26056885719299316, "eval/total_success_rate_ub_0.05": 0.5018224716186523, "eval/total_success_rate_lb_0.1": 0.2753894329071045, "eval/total_success_rate_ub_0.1": 0.48588883876800537} \ No newline at end of file diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/logs/debug-internal.log b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5f096e6cfe91ddf6e6e9cd5c00ff35d6035ad080 --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/logs/debug-internal.log @@ -0,0 +1,767 @@ +2025-01-07 17:04:15,935 INFO StreamThr :2193794 [internal.py:wandb_internal():85] W&B internal server running at pid: 2193794, started at: 2025-01-07 17:04:15.935397 +2025-01-07 17:04:15,936 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:04:15,947 INFO WriterThread:2193794 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/run-9hy9qnsj.wandb +2025-01-07 17:04:15,948 DEBUG SenderThread:2193794 [sender.py:send():379] send: header +2025-01-07 17:04:15,959 DEBUG SenderThread:2193794 [sender.py:send():379] send: run +2025-01-07 17:04:16,202 INFO SenderThread:2193794 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files +2025-01-07 17:04:16,203 INFO SenderThread:2193794 [sender.py:_start_run_threads():1188] run started: 9hy9qnsj with start time 1736287455.937062 +2025-01-07 17:04:16,220 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:04:16,220 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:04:16,293 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:04:16,298 DEBUG HandlerThread:2193794 [system_info.py:__init__():26] System info init +2025-01-07 17:04:16,298 DEBUG HandlerThread:2193794 [system_info.py:__init__():41] System info init done +2025-01-07 17:04:16,298 INFO HandlerThread:2193794 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:04:16,298 INFO SystemMonitor:2193794 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:04:16,298 INFO HandlerThread:2193794 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:04:16,299 INFO SystemMonitor:2193794 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:04:16,300 INFO SystemMonitor:2193794 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:04:16,300 INFO SystemMonitor:2193794 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:04:16,301 INFO SystemMonitor:2193794 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:04:16,301 INFO SystemMonitor:2193794 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:04:16,311 DEBUG HandlerThread:2193794 [system_info.py:probe():152] Probing system +2025-01-07 17:04:16,312 DEBUG HandlerThread:2193794 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:04:16,317 DEBUG HandlerThread:2193794 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:04:16,317 DEBUG HandlerThread:2193794 [system_info.py:probe():200] Probing system done +2025-01-07 17:04:16,317 DEBUG HandlerThread:2193794 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:04:16.311164', 'startedAt': '2025-01-07T22:04:15.927913', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.31675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.575, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.433, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 730.4026374816895}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:04:16,317 INFO HandlerThread:2193794 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:04:16,317 INFO HandlerThread:2193794 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:04:16,317 DEBUG HandlerThread:2193794 [system_info.py:_save_code():44] Saving code +2025-01-07 17:04:16,319 DEBUG HandlerThread:2193794 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:04:16,319 DEBUG HandlerThread:2193794 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:04:16,383 DEBUG HandlerThread:2193794 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:04:16,384 INFO HandlerThread:2193794 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:04:16,387 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:04:16,387 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:04:16,387 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:04:16,387 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:04:16,622 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:04:16,622 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:04:16,622 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:04:16,623 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:04:16,624 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:16,641 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/my2aa3w4-wandb-metadata.json +2025-01-07 17:04:16,654 INFO wandb-upload_2:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/f7vytrbm-diff.patch +2025-01-07 17:04:16,658 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/ry4chh2f-code/FISH/eval_robot.py +2025-01-07 17:04:16,665 INFO HandlerThread:2193794 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/tb" + save: true +} +control { + relay_id: "7fa4d6ccfc7f47bdb9db499271e3d46c" +} +_info { + stream_id: "9hy9qnsj" +} + +2025-01-07 17:04:16,665 DEBUG HandlerThread:2193794 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:04:16,684 DEBUG SenderThread:2193794 [sender.py:send():379] send: telemetry +2025-01-07 17:04:16,716 DEBUG SenderThread:2193794 [sender.py:send():379] send: tbrecord +2025-01-07 17:04:16,717 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:04:16,717 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 with policy live +2025-01-07 17:04:16,913 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/ke0wobei-events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:04:17,204 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/diff.patch +2025-01-07 17:04:17,204 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:17,204 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:04:17,204 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/requirements.txt +2025-01-07 17:04:17,205 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/code/FISH/eval_robot.py +2025-01-07 17:04:17,205 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-metadata.json +2025-01-07 17:04:17,205 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/code/FISH +2025-01-07 17:04:17,205 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/code +2025-01-07 17:04:17,624 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:18,624 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:19,203 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:19,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:20,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:21,165 DEBUG SenderThread:2193794 [sender.py:send():379] send: config +2025-01-07 17:04:21,165 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:21,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:22,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:23,205 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:23,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:24,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:25,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:26,623 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:26,625 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:27,206 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:27,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:28,645 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:29,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:30,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:31,207 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:31,622 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:04:31,623 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:04:31,665 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:31,780 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:32,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:33,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:34,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:35,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:36,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:37,624 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:37,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:38,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:39,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:40,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:41,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:42,624 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:42,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:43,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:44,628 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:45,628 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:46,623 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:04:46,623 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:04:46,665 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:47,628 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:47,750 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:48,211 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/config.yaml +2025-01-07 17:04:48,628 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:49,629 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:50,628 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:51,212 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:51,629 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:52,629 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:53,213 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:53,526 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:53,630 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:54,630 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:55,214 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:04:55,630 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:56,630 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:57,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:58,624 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:04:58,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:04:59,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:00,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:01,623 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:05:01,623 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:05:01,665 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:02,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:03,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:03,776 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:04,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:05,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:06,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:07,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:08,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:09,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:09,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:10,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:11,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:12,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:13,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:14,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:14,909 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:15,219 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:15,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:16,301 DEBUG SystemMonitor:2193794 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:05:16,302 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:05:16,623 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:05:16,623 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:05:16,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:17,220 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:17,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:18,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:19,220 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:19,636 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:20,054 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:20,299 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:05:20,299 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 with policy now +2025-01-07 17:05:20,300 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:05:20,300 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:05:20,300 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:05:20,301 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:05:20,530 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/pm3lgumd-media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 +2025-01-07 17:05:20,636 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:21,221 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:21,221 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 +2025-01-07 17:05:21,221 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:05:21,221 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos +2025-01-07 17:05:21,222 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media +2025-01-07 17:05:21,222 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval +2025-01-07 17:05:21,501 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/80ci2ce8-media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 +2025-01-07 17:05:21,636 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:22,637 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:23,222 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:23,661 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:24,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:25,222 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:25,636 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:25,662 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:26,636 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:27,636 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:28,637 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:29,224 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:29,637 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:30,637 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:31,623 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:05:31,623 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:05:31,665 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:31,722 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:32,637 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:33,637 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:34,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:35,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:36,226 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:05:36,469 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/vh74ol48-events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:05:36,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:36,812 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:37,226 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:37,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:38,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:39,227 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:39,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:40,638 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:41,228 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:05:41,639 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:42,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:42,641 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:43,641 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:44,641 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:45,641 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:46,302 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:05:46,623 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:05:46,623 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:05:46,665 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:47,641 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:47,739 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:48,641 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:49,644 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:50,644 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:51,644 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:52,644 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:53,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:53,645 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:54,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:55,646 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:56,646 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:57,646 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:58,627 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:05:58,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:05:59,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:00,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:01,633 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:06:01,634 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:06:01,677 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:02,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:03,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:04,099 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:04,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:05,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:06,647 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:07,615 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:06:07,615 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 with policy now +2025-01-07 17:06:07,618 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:06:07,618 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:06:07,618 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:06:07,619 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:06:07,648 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:07,838 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/d3w0153m-media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 +2025-01-07 17:06:08,238 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:06:08,238 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 +2025-01-07 17:06:08,238 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval +2025-01-07 17:06:08,510 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/k9jm2vg4-media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 +2025-01-07 17:06:08,649 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:09,238 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:09,636 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:09,653 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:10,650 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:11,651 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:12,652 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:13,240 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:13,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:14,654 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:14,737 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:15,240 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:15,654 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:16,303 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:06:16,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:06:16,632 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:06:16,673 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:17,655 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:18,654 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:19,655 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:19,804 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:20,655 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:21,655 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:22,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:23,655 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:24,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:25,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:25,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:26,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:27,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:28,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:29,097 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:06:29,097 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:06:29,097 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:06:29,245 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:06:29,245 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:06:29,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:30,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:31,246 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:31,513 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:31,630 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:06:31,631 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:06:31,677 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:32,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:33,246 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:33,656 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:34,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:35,247 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:35,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:36,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:36,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:37,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:38,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:39,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:40,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:41,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:41,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:42,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:43,657 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:44,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:45,250 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:45,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:46,303 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:06:46,633 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:06:46,633 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:06:46,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:46,778 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:47,251 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:47,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:48,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:49,252 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:49,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:50,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:51,252 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:51,658 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:52,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:52,659 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:53,253 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:53,659 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:54,659 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:55,254 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:06:55,659 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:56,659 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:57,659 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:58,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:06:58,659 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:06:59,660 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:00,513 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:07:00,513 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 with policy now +2025-01-07 17:07:00,514 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:07:00,515 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:07:00,515 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:07:00,515 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:07:00,665 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:00,763 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/txi82k5l-media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 +2025-01-07 17:07:01,256 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:07:01,256 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:01,256 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 +2025-01-07 17:07:01,256 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval +2025-01-07 17:07:01,527 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/tq2ny0zk-media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 +2025-01-07 17:07:01,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:07:01,636 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:07:01,677 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:02,664 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:03,256 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:03,665 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:03,772 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:04,671 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:05,257 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:05,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:06,689 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:07,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:08,674 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:09,259 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:09,676 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:09,690 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:10,259 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:07:10,493 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/7xee8ci5-events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:07:10,681 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:10,935 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:07:10,935 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:07:10,936 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:07:11,259 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:07:11,260 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:11,675 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:12,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:13,260 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:13,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:14,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:15,105 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:15,261 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:15,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:16,304 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:07:16,631 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:07:16,632 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:07:16,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:17,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:18,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:19,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:20,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:20,738 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:21,678 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:22,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:23,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:24,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:25,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:26,634 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:26,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:27,679 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:28,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:29,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:30,685 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:31,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:07:31,632 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:07:31,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:31,790 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:32,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:33,267 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:33,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:34,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:35,680 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:36,681 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:37,635 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:37,681 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:38,681 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:39,681 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:40,681 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:41,725 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:42,007 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:07:42,008 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 with policy now +2025-01-07 17:07:42,018 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:07:42,019 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:07:42,019 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:07:42,020 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:07:42,260 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/re5ddgss-media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 +2025-01-07 17:07:42,270 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:07:42,270 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 +2025-01-07 17:07:42,270 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval +2025-01-07 17:07:42,527 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/njy744f5-media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 +2025-01-07 17:07:42,692 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:43,060 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:43,270 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:43,691 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:44,695 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:45,689 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:46,304 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:07:46,639 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:07:46,639 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:07:46,696 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:47,272 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:47,692 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:48,181 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:48,696 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:49,273 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:49,701 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:50,694 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:51,705 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:52,706 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:52,989 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:07:52,989 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:07:52,990 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:07:53,274 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:07:53,274 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:07:53,707 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:53,783 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:54,700 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:55,275 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:55,705 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:56,701 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:57,703 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:58,700 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:07:59,276 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:07:59,626 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:07:59,700 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:00,700 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:01,277 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:01,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:08:01,632 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:08:01,700 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:02,701 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:03,700 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:04,701 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:05,605 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:05,701 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:06,707 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:07,279 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:07,707 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:08,707 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:09,708 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:10,709 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:11,280 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:11,605 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:11,709 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:12,709 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:13,713 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:14,713 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:15,713 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:16,305 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:08:16,632 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:08:16,632 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:08:16,713 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:16,784 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:17,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:18,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:19,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:20,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:21,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:22,639 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:22,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:23,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:24,714 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:25,719 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:26,535 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:08:26,535 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 with policy now +2025-01-07 17:08:26,536 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:08:26,536 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:08:26,536 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:26,537 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:26,721 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:26,758 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/nso8q348-media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 +2025-01-07 17:08:27,286 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:08:27,286 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:27,286 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 +2025-01-07 17:08:27,286 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval +2025-01-07 17:08:27,586 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/yy3gk1va-media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 +2025-01-07 17:08:27,646 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:27,724 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:28,728 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:29,287 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:29,731 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:30,728 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:31,287 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:31,644 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:08:31,644 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:08:31,729 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:32,726 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:32,735 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:33,739 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:34,741 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:35,289 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:35,289 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:08:35,517 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/cm3kasi4-events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:08:35,731 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:36,043 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:08:36,043 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:36,043 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:36,160 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:08:36,160 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:08:36,289 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:08:36,289 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:08:36,289 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media +2025-01-07 17:08:36,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table +2025-01-07 17:08:36,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval +2025-01-07 17:08:36,304 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:08:36,460 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/vtqzykfj-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:08:36,682 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:08:36,691 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:08:36,728 INFO wandb-upload_1:2193794 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphq5633y_ +2025-01-07 17:08:36,731 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:37,183 INFO SenderThread:2193794 [sender.py:send_request_log_artifact():1525] logged artifact run-9hy9qnsj-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTg3OTkwOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY4NzU3NQ==', 'latestArtifact': None}} +2025-01-07 17:08:37,183 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:08:37,183 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 17:08:37,183 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:08:37,183 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,184 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,184 DEBUG SenderThread:2193794 [sender.py:send():379] send: telemetry +2025-01-07 17:08:37,184 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,188 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,188 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,189 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,189 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,189 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,189 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,190 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,190 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,190 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,190 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,191 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,191 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,191 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,191 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,191 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,192 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,192 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,192 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,192 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,192 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:37,193 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:37,193 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:08:37,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:08:37,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:37,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:08:37,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json +2025-01-07 17:08:37,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval +2025-01-07 17:08:37,427 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/yzjsydde-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:08:37,517 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/itcbvo4b-media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json +2025-01-07 17:08:37,532 INFO wandb-upload_2:2193794 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpio2exkdw +2025-01-07 17:08:37,731 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:37,947 INFO SenderThread:2193794 [sender.py:send_request_log_artifact():1525] logged artifact run-9hy9qnsj-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTg3OTk2Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY4NzU3OQ==', 'latestArtifact': None}} +2025-01-07 17:08:37,951 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:38,048 DEBUG SenderThread:2193794 [sender.py:send():379] send: files +2025-01-07 17:08:38,048 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json with policy now +2025-01-07 17:08:38,048 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:08:38,048 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:38,049 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:38,280 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/osijn9k6-media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json +2025-01-07 17:08:38,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:08:38,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/config.yaml +2025-01-07 17:08:38,369 DEBUG SenderThread:2193794 [sender.py:send():379] send: exit +2025-01-07 17:08:38,370 INFO SenderThread:2193794 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:08:38,370 INFO SenderThread:2193794 [sender.py:send_exit():588] handling runtime: 262 +2025-01-07 17:08:38,370 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:38,370 INFO SenderThread:2193794 [sender.py:send_exit():594] send defer +2025-01-07 17:08:38,370 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:38,370 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:08:38,370 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:38,371 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:08:38,371 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:08:38,371 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:38,371 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:08:38,371 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:38,371 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:08:38,371 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:08:38,371 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:38,371 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:08:38,371 INFO HandlerThread:2193794 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:08:38,371 DEBUG SystemMonitor:2193794 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:08:38,371 DEBUG SystemMonitor:2193794 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:08:38,372 INFO HandlerThread:2193794 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:08:38,372 INFO HandlerThread:2193794 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:08:38,425 INFO HandlerThread:2193794 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:08:38,425 INFO HandlerThread:2193794 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:08:38,425 INFO HandlerThread:2193794 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:08:38,425 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:38,425 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:08:38,426 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:08:38,426 DEBUG SenderThread:2193794 [sender.py:send():379] send: stats +2025-01-07 17:08:38,426 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:38,426 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:08:38,426 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:38,426 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:08:38,426 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:08:38,426 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:38,426 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:08:39,290 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:08:39,291 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:45,049 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:08:45,049 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:45,049 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:08:45,050 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:45,050 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:08:45,050 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:08:45,051 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:45,051 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:08:45,051 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:08:45,051 DEBUG SenderThread:2193794 [sender.py:send():379] send: history +2025-01-07 17:08:45,051 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:08:45,052 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:45,052 DEBUG SenderThread:2193794 [sender.py:send():379] send: summary +2025-01-07 17:08:45,056 INFO SenderThread:2193794 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:08:45,056 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:45,056 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:08:45,056 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:08:45,056 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:45,056 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:08:45,057 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:45,057 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:08:45,057 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:08:45,057 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:08:45,057 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:45,057 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:08:45,057 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:45,057 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:08:45,293 INFO Thread-12 :2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:08:45,371 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:08:48,591 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:08:48,591 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:08:48,592 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:48,592 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:08:48,592 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:48,592 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:08:48,592 INFO SenderThread:2193794 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:08:48,594 INFO SenderThread:2193794 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:08:48,596 INFO SenderThread:2193794 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:08:48,607 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:08:48,607 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:48,607 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:08:48,607 DEBUG SenderThread:2193794 [sender.py:send():379] send: artifact +2025-01-07 17:08:49,091 INFO SenderThread:2193794 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'versionIndex': 434}}} +2025-01-07 17:08:49,091 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:49,091 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:08:49,091 INFO SenderThread:2193794 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:08:49,294 INFO SenderThread:2193794 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:49,294 INFO SenderThread:2193794 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files +2025-01-07 17:08:49,294 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/config.yaml config.yaml +2025-01-07 17:08:49,294 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json wandb-summary.json +2025-01-07 17:08:49,294 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:08:49,295 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/requirements.txt requirements.txt +2025-01-07 17:08:49,295 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:08:49,295 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log output.log +2025-01-07 17:08:49,295 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/diff.patch diff.patch +2025-01-07 17:08:49,297 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:08:49,299 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 media/videos/eval/1_eval_1_1c9d0d50c4ae77895081.mp4 +2025-01-07 17:08:49,299 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 media/videos/eval/4_eval_4_f56992d8041cb7cbbc74.mp4 +2025-01-07 17:08:49,299 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 media/videos/eval/3_eval_3_7f865df86cb2d9b9e846.mp4 +2025-01-07 17:08:49,299 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 media/videos/eval/0_eval_0_3304018f8dfdbea23d5c.mp4 +2025-01-07 17:08:49,299 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 media/videos/eval/2_eval_2_21ea113d4c8065f9636b.mp4 +2025-01-07 17:08:49,304 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 17:08:49,304 INFO SenderThread:2193794 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json media/table/eval/total_success_rate_ci_6_74d0bc1c5ec1209580e1.table.json +2025-01-07 17:08:49,304 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:08:49,304 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:49,304 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:08:49,304 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:49,305 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:08:49,305 INFO SenderThread:2193794 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:08:49,372 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:08:49,372 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:08:49,467 INFO wandb-upload_1:2193794 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/wandb-summary.json +2025-01-07 17:08:49,486 INFO wandb-upload_2:2193794 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/config.yaml +2025-01-07 17:08:49,525 INFO wandb-upload_0:2193794 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/requirements.txt +2025-01-07 17:08:49,533 INFO wandb-upload_3:2193794 [upload_job.py:push():130] Uploaded file /tmp/tmpkpyv8g01wandb/rhjt2y7r-events.out.tfevents.1736287456.leonmkim-ROG-Strix-G15CS-G15CS.2193627.0 +2025-01-07 17:08:49,562 INFO wandb-upload_4:2193794 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/files/output.log +2025-01-07 17:08:49,762 INFO Thread-11 (_thread_body):2193794 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:08:49,762 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:49,762 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:08:49,762 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:49,762 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:08:49,763 INFO SenderThread:2193794 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:08:49,764 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:08:49,764 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:49,764 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:08:49,764 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:49,764 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:08:49,764 INFO SenderThread:2193794 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:08:49,844 INFO SenderThread:2193794 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:08:49,845 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:08:49,845 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:49,845 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:08:49,845 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:49,845 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:08:49,845 INFO SenderThread:2193794 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:08:49,845 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:08:49,845 INFO HandlerThread:2193794 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:08:49,845 DEBUG SenderThread:2193794 [sender.py:send():379] send: final +2025-01-07 17:08:49,846 DEBUG SenderThread:2193794 [sender.py:send():379] send: footer +2025-01-07 17:08:49,846 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: defer +2025-01-07 17:08:49,846 INFO SenderThread:2193794 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:08:49,846 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:08:49,846 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:08:49,847 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:08:49,847 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:08:49,847 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:08:49,847 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:08:49,847 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:08:49,849 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:08:49,907 DEBUG SenderThread:2193794 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:08:49,908 INFO MainThread:2193794 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:08:49,908 INFO MainThread:2193794 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:08:49,908 INFO MainThread:2193794 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:08:49,908 DEBUG HandlerThread:2193794 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:08:49,909 INFO HandlerThread:2193794 [handler.py:finish():882] shutting down handler +2025-01-07 17:08:50,847 INFO WriterThread:2193794 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/run-9hy9qnsj.wandb +2025-01-07 17:08:50,908 INFO SenderThread:2193794 [sender.py:finish():1615] shutting down sender +2025-01-07 17:08:50,908 INFO SenderThread:2193794 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:08:50,908 INFO SenderThread:2193794 [file_pusher.py:join():175] waiting for file pusher diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/logs/debug.log b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..09ef84a31b46b5d43251a740170af934f6f08957 --- /dev/null +++ b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Configure stats pid to 2193627 +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/settings +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/logs/debug.log +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/logs/debug-internal.log +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj'} +2025-01-07 17:04:15,930 INFO MainThread:2193627 [wandb_init.py:init():619] starting backend +2025-01-07 17:04:15,931 INFO MainThread:2193627 [wandb_init.py:init():623] setting up manager +2025-01-07 17:04:15,934 INFO MainThread:2193627 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:04:15,936 INFO MainThread:2193627 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:04:15,947 INFO MainThread:2193627 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:04:15,956 INFO MainThread:2193627 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:04:16,219 INFO MainThread:2193627 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:04:16,286 INFO MainThread:2193627 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:04:16,286 INFO MainThread:2193627 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:04:16,624 INFO MainThread:2193627 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:04:16,624 INFO MainThread:2193627 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:04:16,624 INFO MainThread:2193627 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:04:16,624 INFO MainThread:2193627 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:04:16,626 INFO MainThread:2193627 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:04:16,626 INFO MainThread:2193627 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/9hy9qnsj/tb, True +2025-01-07 17:04:21,164 INFO MainThread:2193627 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 17:08:50,909 WARNING MsgRouterThr:2193627 [router.py:message_loop():77] message_loop has been closed diff --git a/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/run-9hy9qnsj.wandb b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/run-9hy9qnsj.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b6da598dfdf5e1d27a6c14924642035ec1fb1bc6 Binary files /dev/null and b/9hy9qnsj/wandb/run-20250107_170415-9hy9qnsj/run-9hy9qnsj.wandb differ diff --git a/d4l0r92a/episode_rosbags/episode_1_2025-01-06-17-30-05.bag b/d4l0r92a/episode_rosbags/episode_1_2025-01-06-17-30-05.bag new file mode 100644 index 0000000000000000000000000000000000000000..969336b3db9b094bfb73483a994f64bdd82ef9f4 --- /dev/null +++ b/d4l0r92a/episode_rosbags/episode_1_2025-01-06-17-30-05.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbe7f8bfbdd933bc0388940b84e86fa69bb788d5b4cb4b126d8c035b6e552aa3 +size 1183927957 diff --git a/d4l0r92a/episode_rosbags/episode_2_2025-01-06-17-30-57.bag b/d4l0r92a/episode_rosbags/episode_2_2025-01-06-17-30-57.bag new file mode 100644 index 0000000000000000000000000000000000000000..1e60feed01311a1aeb7d5e2f018bd30cebacf90c --- /dev/null +++ b/d4l0r92a/episode_rosbags/episode_2_2025-01-06-17-30-57.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ecfa81cad78ff67ceb5db08776e4c930b8506fdd01c00a5677050448d328ce8 +size 1411768229 diff --git a/d4l0r92a/episode_rosbags/episode_3_2025-01-06-17-31-41.bag b/d4l0r92a/episode_rosbags/episode_3_2025-01-06-17-31-41.bag new file mode 100644 index 0000000000000000000000000000000000000000..28a80fc5da6b77d844d5a69ad3fd9d493e65609b --- /dev/null +++ b/d4l0r92a/episode_rosbags/episode_3_2025-01-06-17-31-41.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcf4b000d3c9fa2d2d6491f90e78a7e03a2d6a368703be8afbefffaa130213a2 +size 1088677680 diff --git a/d4l0r92a/episode_rosbags/episode_4_2025-01-06-17-32-27.bag b/d4l0r92a/episode_rosbags/episode_4_2025-01-06-17-32-27.bag new file mode 100644 index 0000000000000000000000000000000000000000..e986487404c153ae403a74cf1b65b66f100a08cd --- /dev/null +++ b/d4l0r92a/episode_rosbags/episode_4_2025-01-06-17-32-27.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c4ee5d9a7df68c0ce7069ab671548d8adfe95881c08da4cef9309412a4247bd +size 1411809418 diff --git a/i3pbiz9p/episode_rosbags/episode_0_2025-01-06-17-34-00.bag b/i3pbiz9p/episode_rosbags/episode_0_2025-01-06-17-34-00.bag new file mode 100644 index 0000000000000000000000000000000000000000..db5018b6dfcc719aefbcfe52b3ad571d7b2fc190 --- /dev/null +++ b/i3pbiz9p/episode_rosbags/episode_0_2025-01-06-17-34-00.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09a51ae73887098451af7c0696ab1b76dd6d92c7a2b53cf143fb85a2d8222058 +size 1409844921 diff --git a/i3pbiz9p/episode_rosbags/episode_2_2025-01-06-17-35-57.bag b/i3pbiz9p/episode_rosbags/episode_2_2025-01-06-17-35-57.bag new file mode 100644 index 0000000000000000000000000000000000000000..513d75719c4a215d64aff9141dd0ff027eb48a4b --- /dev/null +++ b/i3pbiz9p/episode_rosbags/episode_2_2025-01-06-17-35-57.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cc0d9c21c583bcc9684201c45bad8fc0e7d1e12eb435c80cf98a741f6bc3350 +size 1412194823 diff --git a/i3pbiz9p/episode_rosbags/episode_3_2025-01-06-17-36-38.bag b/i3pbiz9p/episode_rosbags/episode_3_2025-01-06-17-36-38.bag new file mode 100644 index 0000000000000000000000000000000000000000..3e194bffcff9f47f9ffc9045a6e5c2520e10d2f5 --- /dev/null +++ b/i3pbiz9p/episode_rosbags/episode_3_2025-01-06-17-36-38.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4549a5c3525678431ecd417c3a569ae06d3fbc29cce61363e7967ce9b86edef +size 1413796778 diff --git a/i3pbiz9p/episode_rosbags/episode_4_2025-01-06-17-37-35.bag b/i3pbiz9p/episode_rosbags/episode_4_2025-01-06-17-37-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..cb2214ef522a14c2d910bd8818cd6f94ec76faad --- /dev/null +++ b/i3pbiz9p/episode_rosbags/episode_4_2025-01-06-17-37-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df70910e90a74a3ba2f6c7f54651624a9c2551f797348f1cf89db52eb25a5ff0 +size 1418069946 diff --git a/or3sumxm/.hydra/config.yaml b/or3sumxm/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/or3sumxm/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/or3sumxm/.hydra/hydra.yaml b/or3sumxm/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..690ab36dd9049d6b2080675a9055d40f40e532c1 --- /dev/null +++ b/or3sumxm/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/or3sumxm/episode_rosbags/aligned_depth_to_color_K.npy b/or3sumxm/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/or3sumxm/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/or3sumxm/episode_rosbags/cam_tf_world.npy b/or3sumxm/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/or3sumxm/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/or3sumxm/episode_rosbags/color_K.npy b/or3sumxm/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/or3sumxm/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/or3sumxm/episode_rosbags/depth_K.npy b/or3sumxm/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/or3sumxm/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/or3sumxm/eval_robot.log b/or3sumxm/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..08ae402184bbb75fe5347abc383010fc2f883a86 --- /dev/null +++ b/or3sumxm/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 22:23:02,557][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 22:23:02,559][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/or3sumxm/eval_video/0_eval.mp4 b/or3sumxm/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3724778709f227fb044f0f8dc95696126e7dc671 --- /dev/null +++ b/or3sumxm/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:925e32212f6150e3263f4dba83affde83344996d7e05e7fd3f66cd8cb4a3a004 +size 1114623 diff --git a/or3sumxm/eval_video/1_eval.mp4 b/or3sumxm/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..58d10727d76b99e5e38254cc66738b5b04efe5a8 --- /dev/null +++ b/or3sumxm/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09b93e53fac465967343f77eec42aa5cceb3608921e1b2047787d13e448ac47c +size 1194442 diff --git a/or3sumxm/eval_video/2_eval.mp4 b/or3sumxm/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..726ecd1bb385208c4d7ede75ae6857b536ef8ed7 --- /dev/null +++ b/or3sumxm/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c776c8b0fd9fc136e3726f934b3676dfdd134db8701e6dc63d154ed3ad988b67 +size 1135781 diff --git a/or3sumxm/eval_video/3_eval.mp4 b/or3sumxm/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eb059c7bed31ef462c3c0ea86e35988819f2be79 --- /dev/null +++ b/or3sumxm/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8ac7cd25652f50c9d7e0880db154f37b7d797d1cae5f6df08a106e600210211 +size 1112174 diff --git a/or3sumxm/eval_video/4_eval.mp4 b/or3sumxm/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4f269509c1975d134f5db2d864ceae4a0c7fe81a --- /dev/null +++ b/or3sumxm/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5c5b2ea7b5d6cafa3afbbe5296eba2d129ffab4878ae7389747e6059c97a173 +size 1208248 diff --git a/or3sumxm/tb/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 b/or3sumxm/tb/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 new file mode 100644 index 0000000000000000000000000000000000000000..24b3e86a1f55409e3fa4dcfbf0f6e60ee2a07c7d --- /dev/null +++ b/or3sumxm/tb/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ed8fc76ac576f492edd0004a9d4ffdfafd5f3789b64f88f5bbd83fffe899b41 +size 6539 diff --git a/or3sumxm/wandb/debug-internal.log b/or3sumxm/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..24cc9640e63040f7bf0951d70cbbc5231fdbaff4 --- /dev/null +++ b/or3sumxm/wandb/debug-internal.log @@ -0,0 +1,782 @@ +2025-01-06 22:23:08,456 INFO StreamThr :2124597 [internal.py:wandb_internal():85] W&B internal server running at pid: 2124597, started at: 2025-01-06 22:23:08.456251 +2025-01-06 22:23:08,457 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status +2025-01-06 22:23:08,469 INFO WriterThread:2124597 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/run-or3sumxm.wandb +2025-01-06 22:23:08,470 DEBUG SenderThread:2124597 [sender.py:send():379] send: header +2025-01-06 22:23:08,479 DEBUG SenderThread:2124597 [sender.py:send():379] send: run +2025-01-06 22:23:08,723 INFO SenderThread:2124597 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files +2025-01-06 22:23:08,723 INFO SenderThread:2124597 [sender.py:_start_run_threads():1188] run started: or3sumxm with start time 1736220188.456624 +2025-01-06 22:23:08,740 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 22:23:08,740 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: check_version +2025-01-06 22:23:08,814 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 22:23:08,818 DEBUG HandlerThread:2124597 [system_info.py:__init__():26] System info init +2025-01-06 22:23:08,818 DEBUG HandlerThread:2124597 [system_info.py:__init__():41] System info init done +2025-01-06 22:23:08,818 INFO HandlerThread:2124597 [system_monitor.py:start():194] Starting system monitor +2025-01-06 22:23:08,819 INFO SystemMonitor:2124597 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 22:23:08,819 INFO HandlerThread:2124597 [system_monitor.py:probe():214] Collecting system info +2025-01-06 22:23:08,819 INFO SystemMonitor:2124597 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 22:23:08,820 INFO SystemMonitor:2124597 [interfaces.py:start():188] Started disk monitoring +2025-01-06 22:23:08,820 INFO SystemMonitor:2124597 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 22:23:08,821 INFO SystemMonitor:2124597 [interfaces.py:start():188] Started memory monitoring +2025-01-06 22:23:08,821 INFO SystemMonitor:2124597 [interfaces.py:start():188] Started network monitoring +2025-01-06 22:23:08,831 DEBUG HandlerThread:2124597 [system_info.py:probe():152] Probing system +2025-01-06 22:23:08,833 DEBUG HandlerThread:2124597 [system_info.py:_probe_git():137] Probing git +2025-01-06 22:23:08,840 DEBUG HandlerThread:2124597 [system_info.py:_probe_git():145] Probing git done +2025-01-06 22:23:08,840 DEBUG HandlerThread:2124597 [system_info.py:probe():200] Probing system done +2025-01-06 22:23:08,840 DEBUG HandlerThread:2124597 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:23:08.831861', 'startedAt': '2025-01-07T03:23:08.448550', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3935.8910000000005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.907, 'min': 800.0, 'max': 4700.0}, {'current': 4494.296, 'min': 800.0, 'max': 4700.0}, {'current': 4498.433, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 571.4687652587891}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 22:23:08,840 INFO HandlerThread:2124597 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 22:23:08,840 INFO HandlerThread:2124597 [system_monitor.py:probe():227] Publishing system info +2025-01-06 22:23:08,840 DEBUG HandlerThread:2124597 [system_info.py:_save_code():44] Saving code +2025-01-06 22:23:08,842 DEBUG HandlerThread:2124597 [system_info.py:_save_code():67] Saving code done +2025-01-06 22:23:08,842 DEBUG HandlerThread:2124597 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 22:23:08,908 DEBUG HandlerThread:2124597 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 22:23:08,909 INFO HandlerThread:2124597 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 22:23:08,916 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:23:08,916 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 22:23:08,917 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 22:23:08,917 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 22:23:09,166 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 22:23:09,166 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: python_packages +2025-01-06 22:23:09,166 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:23:09,167 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:23:09,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:09,168 INFO HandlerThread:2124597 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/tb" + save: true +} +control { + relay_id: "c57c9dc407d34f6db0867bc5d8a2d526" +} +_info { + stream_id: "or3sumxm" +} + +2025-01-06 22:23:09,169 DEBUG HandlerThread:2124597 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 22:23:09,175 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/af8fjgre-wandb-metadata.json +2025-01-06 22:23:09,189 INFO wandb-upload_1:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/x3rme31x-code/FISH/eval_robot.py +2025-01-06 22:23:09,199 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/4yasabf7-diff.patch +2025-01-06 22:23:09,254 DEBUG SenderThread:2124597 [sender.py:send():379] send: telemetry +2025-01-06 22:23:09,254 DEBUG SenderThread:2124597 [sender.py:send():379] send: tbrecord +2025-01-06 22:23:09,255 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:23:09,255 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 with policy live +2025-01-06 22:23:09,441 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/7gctc26b-events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:23:09,725 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/diff.patch +2025-01-06 22:23:09,725 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-metadata.json +2025-01-06 22:23:09,725 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/code/FISH/eval_robot.py +2025-01-06 22:23:09,725 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/requirements.txt +2025-01-06 22:23:09,725 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:23:09,725 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:09,726 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/code/FISH +2025-01-06 22:23:09,726 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/code +2025-01-06 22:23:10,167 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:11,167 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:11,724 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:12,167 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:13,167 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:14,167 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:14,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:15,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:15,535 DEBUG SenderThread:2124597 [sender.py:send():379] send: config +2025-01-06 22:23:16,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:17,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:17,726 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:18,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:19,167 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:19,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:20,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:21,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:21,727 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:22,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:23,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:24,166 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:23:24,166 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:23:24,213 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:24,255 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:25,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:25,728 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:26,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:27,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:27,729 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:28,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:29,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:29,729 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:30,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:30,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:31,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:32,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:33,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:34,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:35,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:35,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:36,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:37,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:38,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:39,166 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:23:39,166 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:23:39,213 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:40,173 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:40,271 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:40,732 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/config.yaml +2025-01-06 22:23:41,173 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:42,173 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:43,173 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:44,173 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:45,174 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:46,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:46,175 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:47,175 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:48,175 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:49,217 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:50,175 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:51,176 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:51,413 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:51,735 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:52,176 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:53,176 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:54,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:23:54,168 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:23:54,209 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:55,176 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:56,177 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:56,297 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:23:56,298 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:23:56,298 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 with policy now +2025-01-06 22:23:56,298 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:23:56,298 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:23:56,300 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:23:56,541 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/a7h3c5ij-media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 +2025-01-06 22:23:56,736 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 +2025-01-06 22:23:56,737 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:23:56,737 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos +2025-01-06 22:23:56,737 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media +2025-01-06 22:23:56,737 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval +2025-01-06 22:23:57,006 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/enszts38-media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 +2025-01-06 22:23:57,177 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:57,300 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:57,737 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:58,177 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:59,177 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:00,177 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:01,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:01,738 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:02,177 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:02,593 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:03,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:03,739 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:04,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:05,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:06,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:07,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:08,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:08,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:08,821 DEBUG SystemMonitor:2124597 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 22:24:08,822 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:24:09,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:24:09,168 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:24:09,209 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:10,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:11,179 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:12,179 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:13,179 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:13,236 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:14,179 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:15,179 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:16,180 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:17,180 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:17,743 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:24:17,990 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/6a7gsibc-events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:24:18,180 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:19,073 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:19,180 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:19,744 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:20,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:21,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:21,745 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:22,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:23,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:23,745 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:24,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:24:24,168 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:24:24,213 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:24,306 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:25,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:26,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:27,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:28,182 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:29,183 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:30,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:30,183 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:31,183 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:32,183 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:33,184 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:34,184 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:35,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:35,184 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:36,184 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:37,184 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:38,184 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:38,822 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:24:39,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:24:39,169 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:24:39,209 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:40,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:40,242 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:41,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:42,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:43,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:44,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:45,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:46,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:46,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:47,186 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:48,186 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:48,883 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:24:48,884 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 with policy now +2025-01-06 22:24:48,886 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:24:48,887 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:24:48,887 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:24:48,887 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:24:49,106 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/c5acdw94-media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 +2025-01-06 22:24:49,186 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:49,754 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:49,754 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:24:49,754 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 +2025-01-06 22:24:49,754 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval +2025-01-06 22:24:50,040 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/o8u0uhxz-media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 +2025-01-06 22:24:50,187 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:51,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:51,188 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:51,755 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:52,195 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:53,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:53,755 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:54,175 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:24:54,175 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:24:54,217 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:55,190 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:56,190 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:57,137 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:57,190 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:57,757 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:58,190 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:59,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:00,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:01,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:02,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:02,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:03,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:04,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:05,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:06,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:06,760 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:25:07,192 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:07,413 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:07,605 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:25:07,605 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:25:07,606 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:25:07,760 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:07,760 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:25:08,192 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:08,823 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:25:09,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:25:09,169 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:25:09,213 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:09,761 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:10,198 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:11,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:11,762 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:12,192 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:12,579 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:13,192 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:13,762 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:14,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:15,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:15,763 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:16,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:17,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:18,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:18,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:19,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:20,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:21,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:22,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:23,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:23,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:24,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:25:24,170 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:25:24,213 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:25,194 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:26,194 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:27,194 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:28,195 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:28,315 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:29,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:30,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:31,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:32,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:33,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:34,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:34,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:35,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:36,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:37,197 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:38,197 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:38,823 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:25:38,905 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:25:38,905 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 with policy now +2025-01-06 22:25:38,912 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:25:38,913 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:25:38,913 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:25:38,914 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:25:39,159 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/7tusbm3b-media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 +2025-01-06 22:25:39,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:25:39,170 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:25:39,201 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:39,318 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:39,771 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:39,771 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:25:39,771 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 +2025-01-06 22:25:39,772 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval +2025-01-06 22:25:40,042 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/flny3xxu-media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 +2025-01-06 22:25:40,209 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:40,772 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:41,198 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:41,772 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:42,198 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:43,207 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:43,773 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:44,217 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:45,180 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:45,215 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:46,210 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:47,221 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:47,774 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:48,211 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:49,217 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:50,216 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:51,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:51,215 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:52,216 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:53,216 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:54,175 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:25:54,175 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:25:54,220 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:55,221 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:56,220 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:56,270 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:57,223 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:57,439 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:25:57,439 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:25:57,440 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:25:57,778 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:25:57,778 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:25:58,016 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/9olbj7m2-events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:25:58,220 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:59,222 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:59,779 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:00,224 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:01,218 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:01,289 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:02,218 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:03,218 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:03,780 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:04,218 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:05,218 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:06,218 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:07,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:07,218 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:08,219 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:08,824 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:26:09,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:26:09,172 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:26:09,219 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:10,221 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:11,222 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:12,222 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:12,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:13,229 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:14,229 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:15,229 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:16,229 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:17,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:18,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:18,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:19,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:20,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:21,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:22,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:23,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:23,525 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:23,787 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:24,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:26:24,171 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:26:24,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:25,236 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:26,246 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:26,719 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:26:26,719 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 with policy now +2025-01-06 22:26:26,725 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:26:26,725 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:26:26,725 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:26:26,726 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:26:26,788 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:26:26,788 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 +2025-01-06 22:26:26,788 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval +2025-01-06 22:26:26,940 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/a0t6kgy9-media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 +2025-01-06 22:26:27,208 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/j6iwd27e-media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 +2025-01-06 22:26:27,242 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:27,788 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:28,244 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:28,778 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:28,789 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:29,244 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:30,247 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:31,245 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:31,790 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:32,246 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:33,241 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:33,790 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:33,825 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:34,243 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:35,250 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:36,246 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:37,248 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:38,249 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:38,824 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:26:39,175 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:26:39,175 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:26:39,251 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:39,335 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:40,249 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:41,249 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:42,254 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:43,253 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:44,269 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:45,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:45,256 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:46,256 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:47,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:47,510 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:26:47,510 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:26:47,510 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:26:47,795 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:47,795 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:26:47,795 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:26:48,256 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:49,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:49,796 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:50,187 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:50,259 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:51,255 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:51,796 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:52,255 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:53,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:53,797 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:54,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:26:54,171 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:26:54,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:55,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:55,322 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:56,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:57,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:58,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:59,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:00,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:01,180 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:01,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:02,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:03,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:04,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:05,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:06,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:06,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:07,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:08,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:08,825 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:27:09,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:27:09,173 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:27:09,259 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:10,259 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:11,259 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:11,673 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:11,803 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:12,259 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:13,259 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:14,259 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:15,260 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:16,260 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:17,040 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:17,260 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:17,806 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:18,265 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:19,114 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:27:19,114 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 with policy now +2025-01-06 22:27:19,116 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:27:19,117 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:27:19,117 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:19,117 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:19,268 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:19,344 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/li5x01as-media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 +2025-01-06 22:27:19,807 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:19,808 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:27:19,808 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 +2025-01-06 22:27:19,808 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval +2025-01-06 22:27:20,120 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/tc3miknb-media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 +2025-01-06 22:27:20,272 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:21,272 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:21,808 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:22,190 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:22,277 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:23,276 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:23,809 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:24,182 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:27:24,183 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:27:24,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:25,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:26,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:27,281 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:27,427 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:27,811 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:27,811 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:27:28,056 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/i01uuxaw-events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:27:28,176 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:27:28,176 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:27:28,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:28,402 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:27:28,685 INFO wandb-upload_0:2124597 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbybsquiw +2025-01-06 22:27:28,796 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:27:28,806 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:27:28,811 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:27:28,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 22:27:28,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json +2025-01-06 22:27:28,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media +2025-01-06 22:27:28,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table +2025-01-06 22:27:28,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval +2025-01-06 22:27:29,093 INFO SenderThread:2124597 [sender.py:send_request_log_artifact():1525] logged artifact run-or3sumxm-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU4NDk1Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc0NDUxMw==', 'latestArtifact': None}} +2025-01-06 22:27:29,093 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:27:29,093 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json with policy now +2025-01-06 22:27:29,093 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:27:29,094 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,094 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,095 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:27:29,095 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,095 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,095 DEBUG SenderThread:2124597 [sender.py:send():379] send: telemetry +2025-01-06 22:27:29,095 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,097 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,097 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,098 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,098 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,098 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,098 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,099 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,099 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,099 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,100 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,100 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,100 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,100 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,100 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,101 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,101 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,101 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,101 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,102 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,102 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,102 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,102 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:27:29,272 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/2dly64yz-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 22:27:29,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:29,305 INFO wandb-upload_1:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/mbv4dyit-media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json +2025-01-06 22:27:29,514 INFO wandb-upload_2:2124597 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6qhuco89 +2025-01-06 22:27:29,516 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/tto1q37g-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 22:27:29,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:29,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:27:29,940 INFO SenderThread:2124597 [sender.py:send_request_log_artifact():1525] logged artifact run-or3sumxm-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU4NDk4MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc0NDUxNQ==', 'latestArtifact': None}} +2025-01-06 22:27:29,940 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:27:29,940 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json with policy now +2025-01-06 22:27:29,940 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:27:29,941 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,941 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:30,182 INFO wandb-upload_1:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/ei2jxt2j-media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json +2025-01-06 22:27:30,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:30,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:27:31,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:31,813 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:32,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:32,942 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:33,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:34,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:35,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:36,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:37,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:38,187 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:38,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:38,825 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:27:39,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:27:39,173 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:27:39,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:40,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:41,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:42,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:43,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:43,290 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:43,926 DEBUG SenderThread:2124597 [sender.py:send():379] send: exit +2025-01-06 22:27:43,926 INFO SenderThread:2124597 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 22:27:43,926 INFO SenderThread:2124597 [sender.py:send_exit():588] handling runtime: 275 +2025-01-06 22:27:43,926 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:43,926 INFO SenderThread:2124597 [sender.py:send_exit():594] send defer +2025-01-06 22:27:43,926 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:43,927 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 22:27:43,927 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:43,927 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 22:27:43,927 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 1 +2025-01-06 22:27:43,927 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:43,927 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 22:27:43,927 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:43,927 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 22:27:43,927 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 2 +2025-01-06 22:27:43,927 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:43,927 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 22:27:43,928 INFO HandlerThread:2124597 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 22:27:43,928 DEBUG SystemMonitor:2124597 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 22:27:43,928 DEBUG SystemMonitor:2124597 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 22:27:43,928 INFO HandlerThread:2124597 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 22:27:43,929 INFO HandlerThread:2124597 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 22:27:43,981 INFO HandlerThread:2124597 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 22:27:43,981 INFO HandlerThread:2124597 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 22:27:43,981 INFO HandlerThread:2124597 [interfaces.py:finish():200] Joined network monitor +2025-01-06 22:27:43,982 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:43,982 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 22:27:43,982 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 3 +2025-01-06 22:27:43,982 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:27:43,982 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:43,982 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 22:27:43,983 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:43,983 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 22:27:43,983 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 4 +2025-01-06 22:27:43,983 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:43,983 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 22:27:44,818 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:27:45,818 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:50,565 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:27:50,565 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:50,566 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 22:27:50,567 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:50,567 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 22:27:50,567 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 5 +2025-01-06 22:27:50,567 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:27:50,567 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:50,567 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:27:50,567 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 22:27:50,567 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:50,569 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:50,569 DEBUG SenderThread:2124597 [sender.py:send():379] send: summary +2025-01-06 22:27:50,569 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:50,570 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:50,570 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 22:27:50,570 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 6 +2025-01-06 22:27:50,570 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:50,570 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 22:27:50,570 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:50,570 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 22:27:50,573 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:50,700 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 7 +2025-01-06 22:27:50,700 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:50,700 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 22:27:50,701 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:50,701 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 22:27:50,820 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:27:50,821 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/config.yaml +2025-01-06 22:27:50,927 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:27:52,813 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 8 +2025-01-06 22:27:52,813 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:27:52,813 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:52,813 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 22:27:52,814 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:52,814 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 22:27:52,814 INFO SenderThread:2124597 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 22:27:52,816 INFO SenderThread:2124597 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 22:27:52,818 INFO SenderThread:2124597 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 22:27:52,823 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:52,828 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 9 +2025-01-06 22:27:52,828 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:52,828 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 22:27:52,828 DEBUG SenderThread:2124597 [sender.py:send():379] send: artifact +2025-01-06 22:27:52,928 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:27:53,056 INFO SenderThread:2124597 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 22:27:53,056 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:53,056 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 22:27:53,056 INFO SenderThread:2124597 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 22:27:53,824 INFO SenderThread:2124597 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files +2025-01-06 22:27:53,824 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/config.yaml config.yaml +2025-01-06 22:27:53,824 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json wandb-summary.json +2025-01-06 22:27:53,826 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:27:53,827 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/requirements.txt requirements.txt +2025-01-06 22:27:53,827 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-metadata.json wandb-metadata.json +2025-01-06 22:27:53,828 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log output.log +2025-01-06 22:27:53,828 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/diff.patch diff.patch +2025-01-06 22:27:53,828 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 22:27:53,828 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 +2025-01-06 22:27:53,830 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 +2025-01-06 22:27:53,830 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 +2025-01-06 22:27:53,831 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 +2025-01-06 22:27:53,831 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 +2025-01-06 22:27:53,833 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json +2025-01-06 22:27:53,833 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 22:27:53,833 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 10 +2025-01-06 22:27:53,833 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:27:53,833 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:53,833 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 22:27:53,833 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:53,833 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 22:27:53,833 INFO SenderThread:2124597 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:27:53,928 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:27:53,929 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:27:54,013 INFO wandb-upload_1:2124597 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:27:54,032 INFO wandb-upload_3:2124597 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/requirements.txt +2025-01-06 22:27:54,047 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/config.yaml +2025-01-06 22:27:54,070 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/xkne6xyz-events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:27:54,078 INFO wandb-upload_4:2124597 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:54,279 INFO Thread-11 (_thread_body):2124597 [sender.py:transition_state():614] send defer: 11 +2025-01-06 22:27:54,279 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:54,279 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 22:27:54,279 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:54,279 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 22:27:54,280 INFO SenderThread:2124597 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 22:27:54,281 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 12 +2025-01-06 22:27:54,281 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:54,282 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 22:27:54,282 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:54,282 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 22:27:54,282 INFO SenderThread:2124597 [file_stream.py:finish():601] file stream finish called +2025-01-06 22:27:54,528 INFO SenderThread:2124597 [file_stream.py:finish():605] file stream finish is done +2025-01-06 22:27:54,528 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 13 +2025-01-06 22:27:54,529 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:54,529 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 22:27:54,529 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:54,529 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 22:27:54,529 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 14 +2025-01-06 22:27:54,529 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:54,529 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 22:27:54,529 DEBUG SenderThread:2124597 [sender.py:send():379] send: final +2025-01-06 22:27:54,529 DEBUG SenderThread:2124597 [sender.py:send():379] send: footer +2025-01-06 22:27:54,529 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:54,530 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 22:27:54,530 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:27:54,530 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 22:27:54,530 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:27:54,530 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: server_info +2025-01-06 22:27:54,532 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:27:54,532 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 22:27:54,533 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 22:27:54,533 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:54,589 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:27:54,590 INFO MainThread:2124597 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 22:27:54,590 INFO MainThread:2124597 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 22:27:54,590 INFO MainThread:2124597 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 22:27:54,590 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 22:27:54,591 INFO HandlerThread:2124597 [handler.py:finish():882] shutting down handler +2025-01-06 22:27:55,532 INFO WriterThread:2124597 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/run-or3sumxm.wandb +2025-01-06 22:27:55,590 INFO SenderThread:2124597 [sender.py:finish():1615] shutting down sender +2025-01-06 22:27:55,590 INFO SenderThread:2124597 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:27:55,590 INFO SenderThread:2124597 [file_pusher.py:join():175] waiting for file pusher diff --git a/or3sumxm/wandb/debug.log b/or3sumxm/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..780bd8c9cac4e1fd452cc3d22b4df743b95eb4e5 --- /dev/null +++ b/or3sumxm/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Configure stats pid to 2124431 +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/settings +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/logs/debug.log +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/logs/debug-internal.log +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_init.py:init():569] calling init triggers +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm'} +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_init.py:init():619] starting backend +2025-01-06 22:23:08,452 INFO MainThread:2124431 [wandb_init.py:init():623] setting up manager +2025-01-06 22:23:08,455 INFO MainThread:2124431 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 22:23:08,456 INFO MainThread:2124431 [wandb_init.py:init():631] backend started and connected +2025-01-06 22:23:08,469 INFO MainThread:2124431 [wandb_init.py:init():720] updated telemetry +2025-01-06 22:23:08,477 INFO MainThread:2124431 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 22:23:08,739 INFO MainThread:2124431 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 22:23:08,807 INFO MainThread:2124431 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 22:23:08,807 INFO MainThread:2124431 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 22:23:09,166 INFO MainThread:2124431 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 22:23:09,167 INFO MainThread:2124431 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 22:23:09,167 INFO MainThread:2124431 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 22:23:09,167 INFO MainThread:2124431 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 22:23:09,168 INFO MainThread:2124431 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 22:23:09,168 INFO MainThread:2124431 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/tb, True +2025-01-06 22:23:15,534 INFO MainThread:2124431 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 22:27:55,591 WARNING MsgRouterThr:2124431 [router.py:message_loop():77] message_loop has been closed diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/code/FISH/eval_robot.py b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/config.yaml b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a78a0ccc813f2e9de486fee2a8dbb54e5ce3f45a --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1022_0' +wandb_notes: + desc: null + value: 1022_0_req_1348_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736220188 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/diff.patch b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c28171b0ccbf1f8e5d0d27b96acf339a03de1c06 --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..a719175 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json new file mode 100644 index 0000000000000000000000000000000000000000..cc7e5e91447669fe989ab08b6e5b18934b0ea7ca --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.4, 0.04400911936343024, 0.8621948956718848, 0.05324718365067565, 0.803534984527916, 0.11712844677476308, 0.7487166672447589, 0.23282015328982517, 0.6637700199753331]]} \ No newline at end of file diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json new file mode 100644 index 0000000000000000000000000000000000000000..1727312ef9344802763dccf187a15722c1719312 --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[13, 30, 0.43333333333333335, 0.23517847066453657, 0.6463961004918084, 0.2722032517650341, 0.6243761032570995, 0.2963098884036225, 0.5789882615050649, 0.31285832453536655, 0.562831766890834]]} \ No newline at end of file diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3724778709f227fb044f0f8dc95696126e7dc671 --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:925e32212f6150e3263f4dba83affde83344996d7e05e7fd3f66cd8cb4a3a004 +size 1114623 diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..58d10727d76b99e5e38254cc66738b5b04efe5a8 --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09b93e53fac465967343f77eec42aa5cceb3608921e1b2047787d13e448ac47c +size 1194442 diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..726ecd1bb385208c4d7ede75ae6857b536ef8ed7 --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c776c8b0fd9fc136e3726f934b3676dfdd134db8701e6dc63d154ed3ad988b67 +size 1135781 diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eb059c7bed31ef462c3c0ea86e35988819f2be79 --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8ac7cd25652f50c9d7e0880db154f37b7d797d1cae5f6df08a106e600210211 +size 1112174 diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4f269509c1975d134f5db2d864ceae4a0c7fe81a --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5c5b2ea7b5d6cafa3afbbe5296eba2d129ffab4878ae7389747e6059c97a173 +size 1208248 diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..0c9deb06ec97c362c4abeeee70fcc376779bb147 --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log @@ -0,0 +1,283 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736220195.535275]: resetting environment +[INFO] [1736220195.538775]: cleared current plan +[INFO] [1736220195.540287]: moving to home +[INFO] [1736220196.542636]: reached home +[INFO] [1736220196.543940]: reset action history +[INFO] [1736220199.252541]: environment reset +Starting episode 0 +[INFO] [1736220199.254924]: resetting environment +[INFO] [1736220199.257631]: cleared current plan +[INFO] [1736220199.258881]: moving to home +[INFO] [1736220200.260874]: reached home +[INFO] [1736220200.263390]: reset action history +[INFO] [1736220202.974885]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736220204.558899]: resetting environment +[INFO] [1736220204.565658]: cleared current plan +[INFO] [1736220204.566570]: moving to home +[INFO] [1736220205.569424]: reached home +[INFO] [1736220205.571354]: reset action history +[INFO] [1736220208.277965]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/episode_rosbags/episode_0_2025-01-06-22-23-28.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2124431.log +started roslaunch server http://158.130.50.37:44899/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2124718] +process[print_text-2]: started with pid [2124719] +process[rosbag_record-3]: started with pid [2124720] +started bagging! +For topic gripper_width: timestamp difference is 100473793 for nearest: 1736220229157421740 - target: 1736220229056947947 at idx 249 +For topic gripper_width: timestamp difference is 100606356 for nearest: 1736220229357396040 - target: 1736220229256789684 at idx 249 +[INFO] [1736220235.301392]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736220236.050872]: Stored episode 1. +[INFO] [1736220236.051460]: Saving video... +[INFO] [1736220236.256283]: Video saved! +Starting episode 1 +[INFO] [1736220236.270158]: resetting environment +[INFO] [1736220236.280059]: cleared current plan +[INFO] [1736220236.280280]: moving to home +[INFO] [1736220239.883717]: reached home +[INFO] [1736220239.884484]: reset action history +[INFO] [1736220242.592675]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736220257.025206]: resetting environment +[INFO] [1736220257.028397]: cleared current plan +[INFO] [1736220257.028551]: moving to home +[INFO] [1736220258.029254]: reached home +[INFO] [1736220258.030083]: reset action history +[INFO] [1736220260.737376]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/episode_rosbags/episode_1_2025-01-06-22-24-20.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2124431.log +started roslaunch server http://158.130.50.37:36193/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2124831] +process[print_text-5]: started with pid [2124832] +process[rosbag_record-6]: started with pid [2124856] +started bagging! +[INFO] [1736220287.707921]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1736220288.470375]: Stored episode 2. +[INFO] [1736220288.470737]: Saving video... +[INFO] [1736220288.844337]: Video saved! +Starting episode 2 +[INFO] [1736220288.888753]: resetting environment +[INFO] [1736220288.903369]: cleared current plan +[INFO] [1736220288.903625]: moving to home +[INFO] [1736220292.305471]: reached home +[INFO] [1736220292.310505]: reset action history +[INFO] [1736220295.089370]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736220306.394262]: resetting environment +[INFO] [1736220306.398785]: cleared current plan +[INFO] [1736220306.405027]: moving to home +[INFO] [1736220307.410236]: reached home +[INFO] [1736220307.411930]: reset action history +[INFO] [1736220310.118942]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/episode_rosbags/episode_2_2025-01-06-22-25-10.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2124431.log +started roslaunch server http://158.130.50.37:34983/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2124990] +process[print_text-8]: started with pid [2124991] +process[rosbag_record-9]: started with pid [2124992] +started bagging! +For topic gripper_width: timestamp difference is -198811661 for nearest: 1736220312290759195 - target: 1736220312489570856 at idx 33 +For topic gripper_width: timestamp difference is 101949198 for nearest: 1736220313590678198 - target: 1736220313488729000 at idx 46 +For topic gripper_width: timestamp difference is 102163937 for nearest: 1736220313790728714 - target: 1736220313688564777 at idx 47 +[INFO] [1736220337.351159]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736220338.156145]: Stored episode 3. +[INFO] [1736220338.158457]: Saving video... +[INFO] [1736220338.856246]: Video saved! +Starting episode 3 +[INFO] [1736220338.914999]: resetting environment +[WARN] [1736220338.682847]: Plan exhausted +[WARN] [1736220338.726826]: Plan exhausted +[WARN] [1736220338.762939]: Plan exhausted +[WARN] [1736220338.805773]: Plan exhausted +[WARN] [1736220338.843757]: Plan exhausted +[WARN] [1736220338.884320]: Plan exhausted +[WARN] [1736220338.924281]: Plan exhausted +[INFO] [1736220338.949108]: cleared current plan +[INFO] [1736220338.955177]: moving to home +[INFO] [1736220342.564338]: reached home +[INFO] [1736220342.572838]: reset action history +[INFO] [1736220345.356574]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736220357.270504]: resetting environment +[INFO] [1736220357.281249]: cleared current plan +[INFO] [1736220357.281504]: moving to home +[INFO] [1736220358.290359]: reached home +[INFO] [1736220358.294503]: reset action history +[INFO] [1736220361.074648]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/episode_rosbags/episode_3_2025-01-06-22-26-01.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2124431.log +started roslaunch server http://158.130.50.37:45793/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2125127] +process[print_text-11]: started with pid [2125128] +process[rosbag_record-12]: started with pid [2125152] +started bagging! +For topic gripper_width: timestamp difference is -107673019 for nearest: 1736220382290729195 - target: 1736220382398402214 at idx 249 +preempting episode +[INFO] [1736220385.279231]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1736220386.035989]: Stored episode 4. +[INFO] [1736220386.044132]: Saving video... +[INFO] [1736220386.666753]: Video saved! +Starting episode 4 +[INFO] [1736220386.743293]: resetting environment +[INFO] [1736220386.771755]: cleared current plan +[INFO] [1736220386.777838]: moving to home +[WARN] [1736220386.562882]: Plan exhausted +[WARN] [1736220386.603088]: Plan exhausted +[WARN] [1736220386.642846]: Plan exhausted +[WARN] [1736220386.682858]: Plan exhausted +[WARN] [1736220386.724056]: Plan exhausted +[WARN] [1736220386.762837]: Plan exhausted +[INFO] [1736220389.986914]: reached home +[INFO] [1736220389.992076]: reset action history +[INFO] [1736220392.783678]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736220406.981844]: resetting environment +[INFO] [1736220406.999330]: cleared current plan +[INFO] [1736220407.006971]: moving to home +[INFO] [1736220408.011787]: reached home +[INFO] [1736220408.014629]: reset action history +[INFO] [1736220410.796160]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/episode_rosbags/episode_4_2025-01-06-22-26-50.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2124431.log +started roslaunch server http://158.130.50.37:44631/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2125291] +process[print_text-14]: started with pid [2125292] +process[rosbag_record-15]: started with pid [2125293] +started bagging! +For topic gripper_width: timestamp difference is -146845831 for nearest: 1736220430390727268 - target: 1736220430537573099 at idx 249 +For topic gripper_width: timestamp difference is 100630227 for nearest: 1736220437024062339 - target: 1736220436923432112 at idx 249 +[INFO] [1736220438.006850]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736220438.786946]: Stored episode 5. +[INFO] [1736220438.794740]: Saving video... +[INFO] [1736220439.084005]: Video saved! +[INFO] [1736220439.116437]: resetting environment +[INFO] [1736220439.131767]: cleared current plan +[INFO] [1736220439.135664]: moving to home +[INFO] [1736220442.549533]: reached home +[INFO] [1736220442.555948]: reset action history +[INFO] [1736220445.359962]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/requirements.txt b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-metadata.json b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..37494c7bc95dad81e951e1684f59e2754adbce10 --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T03:23:08.831861", + "startedAt": "2025-01-07T03:23:08.448550", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3935.8910000000005, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.907, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4494.296, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.433, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 571.4687652587891 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a4d5c8d33953c3cb8de10b043bc44ba4f9d5ef85 --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "925e32212f6150e3263f4dba83affde83344996d7e05e7fd3f66cd8cb4a3a004", "size": 1114623, "path": "media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4"}, "global_step": 651, "_timestamp": 1736220448.706688, "_runtime": 260.2500638961792, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "09b93e53fac465967343f77eec42aa5cceb3608921e1b2047787d13e448ac47c", "size": 1194442, "path": "media/videos/eval/1_eval_1_09b93e53fac465967343.mp4"}, "eval/num_success": 2.0, "episode": 30.0, "eval/success_rate": 0.4000000059604645, "eval/success_rate_lb_0.01": 0.044009119272232056, "eval/success_rate_ub_0.01": 0.8621948957443237, "eval/success_rate_lb_0.025": 0.05324718356132507, "eval/success_rate_ub_0.025": 0.803534984588623, "eval/success_rate_lb_0.05": 0.11712844669818878, "eval/success_rate_ub_0.05": 0.7487166523933411, "eval/success_rate_lb_0.1": 0.23282015323638916, "eval/success_rate_ub_0.1": 0.6637700200080872, "eval/2_eval": {"_type": "video-file", "sha256": "c776c8b0fd9fc136e3726f934b3676dfdd134db8701e6dc63d154ed3ad988b67", "size": 1135781, "path": "media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "f8ac7cd25652f50c9d7e0880db154f37b7d797d1cae5f6df08a106e600210211", "size": 1112174, "path": "media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "b5c5b2ea7b5d6cafa3afbbe5296eba2d129ffab4878ae7389747e6059c97a173", "size": 1208248, "path": "media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "85c54431b4eb7c8145a72a6437cbffccb84a9cf38051f18b07dd0a638d459f25", "size": 401, "artifact_path": "wandb-client-artifact://n7y2wei26uynuq6z89uxfk2mhya5wp3qvo3cpx0jwphsnh28qpcwrfzc8wcio7izin7ohpa8ynp4cu5kl7s5520t2j4trjptcmx398nf61lobg34cfa6a0xypm3hh6mv/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://biykcukjm0auqv4x2p42b63ni4bpkcze8rpqjq0zd3tm6p903880ae4l1lf84xr0lwf3erv23q0rwhxdn3fbx2ho7trj1m4rwsjrbz54t2vlds5cuhbkf9l2ptwtkg2c:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 13, "total_num_episodes": 30, "total_success_rate": 0.43333333333333335, "total_success_rate_lb_0.01": 0.23517847066453657, "total_success_rate_ub_0.01": 0.6463961004918084, "total_success_rate_lb_0.025": 0.2722032517650341, "total_success_rate_ub_0.025": 0.6243761032570995, "total_success_rate_lb_0.05": 0.2963098884036225, "total_success_rate_ub_0.05": 0.5789882615050649, "total_success_rate_lb_0.1": 0.31285832453536655, "total_success_rate_ub_0.1": 0.562831766890834, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "c76fe457b6d22ff6afdad8a5661a8bd1209afdf21186a00c16ba4393e7c377ca", "size": 522, "artifact_path": "wandb-client-artifact://1b7h5tgqa6h88w3xqm2no2mfaj25hkc1wfucguv91nzec5bbiztcryda76xybzhnii1dkkvo3r3o5pimukce2tc1lf84ialv8ps7va0s0o92o4imuvrn7xc7pzyqeqsm/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://wj0wrdynbn717dtuobtcxen498lg1kf566r82ijjvbbbfm9u90nptdi7jr21nxs2m5c9w8zpjiy30f5cd585wexcgtj3u372f0g4js7a0z5c2jdjxng4997vae7l2dy6:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 275}, "eval/total_success_rate": 0.4333333373069763, "eval/total_success_rate_lb_0.01": 0.23517847061157227, "eval/total_success_rate_ub_0.01": 0.6463961005210876, "eval/total_success_rate_lb_0.025": 0.272203266620636, "eval/total_success_rate_ub_0.025": 0.624376118183136, "eval/total_success_rate_lb_0.05": 0.2963098883628845, "eval/total_success_rate_ub_0.05": 0.578988254070282, "eval/total_success_rate_lb_0.1": 0.31285831332206726, "eval/total_success_rate_ub_0.1": 0.5628317594528198} \ No newline at end of file diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/logs/debug-internal.log b/or3sumxm/wandb/run-20250106_222308-or3sumxm/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..24cc9640e63040f7bf0951d70cbbc5231fdbaff4 --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/logs/debug-internal.log @@ -0,0 +1,782 @@ +2025-01-06 22:23:08,456 INFO StreamThr :2124597 [internal.py:wandb_internal():85] W&B internal server running at pid: 2124597, started at: 2025-01-06 22:23:08.456251 +2025-01-06 22:23:08,457 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status +2025-01-06 22:23:08,469 INFO WriterThread:2124597 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/run-or3sumxm.wandb +2025-01-06 22:23:08,470 DEBUG SenderThread:2124597 [sender.py:send():379] send: header +2025-01-06 22:23:08,479 DEBUG SenderThread:2124597 [sender.py:send():379] send: run +2025-01-06 22:23:08,723 INFO SenderThread:2124597 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files +2025-01-06 22:23:08,723 INFO SenderThread:2124597 [sender.py:_start_run_threads():1188] run started: or3sumxm with start time 1736220188.456624 +2025-01-06 22:23:08,740 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 22:23:08,740 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: check_version +2025-01-06 22:23:08,814 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 22:23:08,818 DEBUG HandlerThread:2124597 [system_info.py:__init__():26] System info init +2025-01-06 22:23:08,818 DEBUG HandlerThread:2124597 [system_info.py:__init__():41] System info init done +2025-01-06 22:23:08,818 INFO HandlerThread:2124597 [system_monitor.py:start():194] Starting system monitor +2025-01-06 22:23:08,819 INFO SystemMonitor:2124597 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 22:23:08,819 INFO HandlerThread:2124597 [system_monitor.py:probe():214] Collecting system info +2025-01-06 22:23:08,819 INFO SystemMonitor:2124597 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 22:23:08,820 INFO SystemMonitor:2124597 [interfaces.py:start():188] Started disk monitoring +2025-01-06 22:23:08,820 INFO SystemMonitor:2124597 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 22:23:08,821 INFO SystemMonitor:2124597 [interfaces.py:start():188] Started memory monitoring +2025-01-06 22:23:08,821 INFO SystemMonitor:2124597 [interfaces.py:start():188] Started network monitoring +2025-01-06 22:23:08,831 DEBUG HandlerThread:2124597 [system_info.py:probe():152] Probing system +2025-01-06 22:23:08,833 DEBUG HandlerThread:2124597 [system_info.py:_probe_git():137] Probing git +2025-01-06 22:23:08,840 DEBUG HandlerThread:2124597 [system_info.py:_probe_git():145] Probing git done +2025-01-06 22:23:08,840 DEBUG HandlerThread:2124597 [system_info.py:probe():200] Probing system done +2025-01-06 22:23:08,840 DEBUG HandlerThread:2124597 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:23:08.831861', 'startedAt': '2025-01-07T03:23:08.448550', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3935.8910000000005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.907, 'min': 800.0, 'max': 4700.0}, {'current': 4494.296, 'min': 800.0, 'max': 4700.0}, {'current': 4498.433, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 571.4687652587891}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 22:23:08,840 INFO HandlerThread:2124597 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 22:23:08,840 INFO HandlerThread:2124597 [system_monitor.py:probe():227] Publishing system info +2025-01-06 22:23:08,840 DEBUG HandlerThread:2124597 [system_info.py:_save_code():44] Saving code +2025-01-06 22:23:08,842 DEBUG HandlerThread:2124597 [system_info.py:_save_code():67] Saving code done +2025-01-06 22:23:08,842 DEBUG HandlerThread:2124597 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 22:23:08,908 DEBUG HandlerThread:2124597 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 22:23:08,909 INFO HandlerThread:2124597 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 22:23:08,916 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:23:08,916 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 22:23:08,917 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 22:23:08,917 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 22:23:09,166 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 22:23:09,166 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: python_packages +2025-01-06 22:23:09,166 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:23:09,167 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:23:09,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:09,168 INFO HandlerThread:2124597 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/tb" + save: true +} +control { + relay_id: "c57c9dc407d34f6db0867bc5d8a2d526" +} +_info { + stream_id: "or3sumxm" +} + +2025-01-06 22:23:09,169 DEBUG HandlerThread:2124597 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 22:23:09,175 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/af8fjgre-wandb-metadata.json +2025-01-06 22:23:09,189 INFO wandb-upload_1:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/x3rme31x-code/FISH/eval_robot.py +2025-01-06 22:23:09,199 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/4yasabf7-diff.patch +2025-01-06 22:23:09,254 DEBUG SenderThread:2124597 [sender.py:send():379] send: telemetry +2025-01-06 22:23:09,254 DEBUG SenderThread:2124597 [sender.py:send():379] send: tbrecord +2025-01-06 22:23:09,255 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:23:09,255 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 with policy live +2025-01-06 22:23:09,441 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/7gctc26b-events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:23:09,725 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/diff.patch +2025-01-06 22:23:09,725 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-metadata.json +2025-01-06 22:23:09,725 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/code/FISH/eval_robot.py +2025-01-06 22:23:09,725 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/requirements.txt +2025-01-06 22:23:09,725 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:23:09,725 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:09,726 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/code/FISH +2025-01-06 22:23:09,726 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/code +2025-01-06 22:23:10,167 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:11,167 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:11,724 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:12,167 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:13,167 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:14,167 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:14,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:15,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:15,535 DEBUG SenderThread:2124597 [sender.py:send():379] send: config +2025-01-06 22:23:16,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:17,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:17,726 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:18,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:19,167 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:19,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:20,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:21,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:21,727 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:22,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:23,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:24,166 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:23:24,166 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:23:24,213 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:24,255 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:25,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:25,728 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:26,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:27,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:27,729 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:28,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:29,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:29,729 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:30,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:30,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:31,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:32,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:33,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:34,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:35,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:35,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:36,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:37,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:38,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:39,166 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:23:39,166 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:23:39,213 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:40,173 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:40,271 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:40,732 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/config.yaml +2025-01-06 22:23:41,173 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:42,173 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:43,173 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:44,173 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:45,174 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:46,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:46,175 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:47,175 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:48,175 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:49,217 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:50,175 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:51,176 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:51,413 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:51,735 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:52,176 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:53,176 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:54,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:23:54,168 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:23:54,209 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:55,176 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:56,177 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:56,297 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:23:56,298 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:23:56,298 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 with policy now +2025-01-06 22:23:56,298 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:23:56,298 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:23:56,300 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:23:56,541 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/a7h3c5ij-media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 +2025-01-06 22:23:56,736 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 +2025-01-06 22:23:56,737 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:23:56,737 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos +2025-01-06 22:23:56,737 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media +2025-01-06 22:23:56,737 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval +2025-01-06 22:23:57,006 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/enszts38-media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 +2025-01-06 22:23:57,177 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:57,300 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:23:57,737 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:23:58,177 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:23:59,177 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:00,177 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:01,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:01,738 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:02,177 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:02,593 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:03,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:03,739 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:04,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:05,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:06,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:07,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:08,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:08,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:08,821 DEBUG SystemMonitor:2124597 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 22:24:08,822 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:24:09,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:24:09,168 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:24:09,209 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:10,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:11,179 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:12,179 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:13,179 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:13,236 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:14,179 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:15,179 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:16,180 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:17,180 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:17,743 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:24:17,990 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/6a7gsibc-events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:24:18,180 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:19,073 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:19,180 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:19,744 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:20,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:21,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:21,745 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:22,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:23,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:23,745 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:24,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:24:24,168 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:24:24,213 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:24,306 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:25,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:26,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:27,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:28,182 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:29,183 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:30,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:30,183 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:31,183 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:32,183 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:33,184 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:34,184 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:35,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:35,184 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:36,184 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:37,184 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:38,184 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:38,822 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:24:39,168 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:24:39,169 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:24:39,209 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:40,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:40,242 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:41,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:42,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:43,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:44,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:45,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:46,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:46,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:47,186 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:48,186 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:48,883 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:24:48,884 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 with policy now +2025-01-06 22:24:48,886 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:24:48,887 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:24:48,887 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:24:48,887 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:24:49,106 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/c5acdw94-media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 +2025-01-06 22:24:49,186 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:49,754 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:49,754 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:24:49,754 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 +2025-01-06 22:24:49,754 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval +2025-01-06 22:24:50,040 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/o8u0uhxz-media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 +2025-01-06 22:24:50,187 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:51,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:51,188 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:51,755 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:52,195 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:53,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:53,755 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:54,175 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:24:54,175 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:24:54,217 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:55,190 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:56,190 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:57,137 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:24:57,190 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:57,757 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:24:58,190 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:24:59,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:00,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:01,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:02,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:02,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:03,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:04,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:05,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:06,191 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:06,760 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:25:07,192 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:07,413 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:07,605 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:25:07,605 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:25:07,606 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:25:07,760 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:07,760 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:25:08,192 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:08,823 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:25:09,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:25:09,169 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:25:09,213 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:09,761 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:10,198 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:11,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:11,762 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:12,192 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:12,579 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:13,192 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:13,762 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:14,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:15,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:15,763 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:16,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:17,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:18,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:18,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:19,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:20,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:21,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:22,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:23,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:23,193 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:24,169 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:25:24,170 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:25:24,213 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:25,194 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:26,194 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:27,194 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:28,195 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:28,315 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:29,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:30,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:31,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:32,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:33,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:34,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:34,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:35,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:36,196 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:37,197 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:38,197 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:38,823 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:25:38,905 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:25:38,905 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 with policy now +2025-01-06 22:25:38,912 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:25:38,913 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:25:38,913 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:25:38,914 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:25:39,159 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/7tusbm3b-media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 +2025-01-06 22:25:39,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:25:39,170 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:25:39,201 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:39,318 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:39,771 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:39,771 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:25:39,771 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 +2025-01-06 22:25:39,772 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval +2025-01-06 22:25:40,042 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/flny3xxu-media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 +2025-01-06 22:25:40,209 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:40,772 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:41,198 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:41,772 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:42,198 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:43,207 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:43,773 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:44,217 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:45,180 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:45,215 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:46,210 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:47,221 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:47,774 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:25:48,211 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:49,217 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:50,216 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:51,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:51,215 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:52,216 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:53,216 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:54,175 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:25:54,175 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:25:54,220 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:55,221 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:56,220 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:56,270 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:25:57,223 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:57,439 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:25:57,439 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:25:57,440 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:25:57,778 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:25:57,778 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:25:58,016 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/9olbj7m2-events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:25:58,220 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:59,222 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:25:59,779 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:00,224 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:01,218 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:01,289 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:02,218 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:03,218 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:03,780 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:04,218 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:05,218 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:06,218 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:07,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:07,218 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:08,219 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:08,824 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:26:09,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:26:09,172 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:26:09,219 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:10,221 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:11,222 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:12,222 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:12,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:13,229 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:14,229 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:15,229 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:16,229 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:17,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:18,178 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:18,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:19,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:20,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:21,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:22,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:23,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:23,525 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:23,787 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:24,170 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:26:24,171 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:26:24,230 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:25,236 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:26,246 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:26,719 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:26:26,719 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 with policy now +2025-01-06 22:26:26,725 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:26:26,725 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:26:26,725 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:26:26,726 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:26:26,788 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:26:26,788 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 +2025-01-06 22:26:26,788 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval +2025-01-06 22:26:26,940 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/a0t6kgy9-media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 +2025-01-06 22:26:27,208 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/j6iwd27e-media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 +2025-01-06 22:26:27,242 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:27,788 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:28,244 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:28,778 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:28,789 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:29,244 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:30,247 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:31,245 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:31,790 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:32,246 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:33,241 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:33,790 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:33,825 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:34,243 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:35,250 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:36,246 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:37,248 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:38,249 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:38,824 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:26:39,175 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:26:39,175 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:26:39,251 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:39,335 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:40,249 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:41,249 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:42,254 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:43,253 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:44,269 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:45,185 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:45,256 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:46,256 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:47,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:47,510 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:26:47,510 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:26:47,510 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:26:47,795 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:47,795 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:26:47,795 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:26:48,256 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:49,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:49,796 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:50,187 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:50,259 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:51,255 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:51,796 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:52,255 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:53,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:53,797 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:26:54,171 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:26:54,171 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:26:54,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:55,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:55,322 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:26:56,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:57,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:58,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:26:59,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:00,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:01,180 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:01,257 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:02,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:03,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:04,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:05,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:06,181 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:06,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:07,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:08,258 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:08,825 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:27:09,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:27:09,173 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:27:09,259 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:10,259 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:11,259 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:11,673 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:11,803 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:12,259 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:13,259 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:14,259 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:15,260 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:16,260 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:17,040 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:17,260 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:17,806 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:18,265 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:19,114 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:27:19,114 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 with policy now +2025-01-06 22:27:19,116 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:27:19,117 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:27:19,117 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:19,117 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:19,268 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:19,344 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/li5x01as-media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 +2025-01-06 22:27:19,807 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:19,808 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:27:19,808 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 +2025-01-06 22:27:19,808 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval +2025-01-06 22:27:20,120 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/tc3miknb-media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 +2025-01-06 22:27:20,272 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:21,272 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:21,808 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:22,190 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:22,277 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:23,276 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:23,809 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:24,182 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:27:24,183 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:27:24,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:25,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:26,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:27,281 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:27,427 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:27,811 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:27,811 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:27:28,056 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/i01uuxaw-events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:27:28,176 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:27:28,176 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:27:28,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:28,402 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:27:28,685 INFO wandb-upload_0:2124597 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbybsquiw +2025-01-06 22:27:28,796 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:27:28,806 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:27:28,811 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:27:28,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 22:27:28,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json +2025-01-06 22:27:28,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media +2025-01-06 22:27:28,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table +2025-01-06 22:27:28,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval +2025-01-06 22:27:29,093 INFO SenderThread:2124597 [sender.py:send_request_log_artifact():1525] logged artifact run-or3sumxm-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU4NDk1Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc0NDUxMw==', 'latestArtifact': None}} +2025-01-06 22:27:29,093 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:27:29,093 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json with policy now +2025-01-06 22:27:29,093 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:27:29,094 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,094 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,095 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:27:29,095 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,095 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,095 DEBUG SenderThread:2124597 [sender.py:send():379] send: telemetry +2025-01-06 22:27:29,095 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,097 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,097 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,098 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,098 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,098 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,098 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,099 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,099 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,099 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,100 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,100 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,100 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,100 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,100 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,101 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,101 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,101 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,101 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,102 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,102 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,102 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:29,102 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:27:29,272 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/2dly64yz-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 22:27:29,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:29,305 INFO wandb-upload_1:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/mbv4dyit-media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json +2025-01-06 22:27:29,514 INFO wandb-upload_2:2124597 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6qhuco89 +2025-01-06 22:27:29,516 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/tto1q37g-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 22:27:29,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:29,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:27:29,940 INFO SenderThread:2124597 [sender.py:send_request_log_artifact():1525] logged artifact run-or3sumxm-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU4NDk4MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc0NDUxNQ==', 'latestArtifact': None}} +2025-01-06 22:27:29,940 DEBUG SenderThread:2124597 [sender.py:send():379] send: files +2025-01-06 22:27:29,940 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json with policy now +2025-01-06 22:27:29,940 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:27:29,941 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:29,941 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:30,182 INFO wandb-upload_1:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/ei2jxt2j-media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json +2025-01-06 22:27:30,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:30,812 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:27:31,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:31,813 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:32,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:32,942 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:33,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:34,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:35,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:36,273 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:37,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:38,187 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:38,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:38,825 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:27:39,172 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:27:39,173 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:27:39,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:40,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:41,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:42,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:43,274 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:43,290 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:43,926 DEBUG SenderThread:2124597 [sender.py:send():379] send: exit +2025-01-06 22:27:43,926 INFO SenderThread:2124597 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 22:27:43,926 INFO SenderThread:2124597 [sender.py:send_exit():588] handling runtime: 275 +2025-01-06 22:27:43,926 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:43,926 INFO SenderThread:2124597 [sender.py:send_exit():594] send defer +2025-01-06 22:27:43,926 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:43,927 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 22:27:43,927 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:43,927 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 22:27:43,927 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 1 +2025-01-06 22:27:43,927 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:43,927 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 22:27:43,927 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:43,927 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 22:27:43,927 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 2 +2025-01-06 22:27:43,927 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:43,927 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 22:27:43,928 INFO HandlerThread:2124597 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 22:27:43,928 DEBUG SystemMonitor:2124597 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 22:27:43,928 DEBUG SystemMonitor:2124597 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 22:27:43,928 INFO HandlerThread:2124597 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 22:27:43,929 INFO HandlerThread:2124597 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 22:27:43,981 INFO HandlerThread:2124597 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 22:27:43,981 INFO HandlerThread:2124597 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 22:27:43,981 INFO HandlerThread:2124597 [interfaces.py:finish():200] Joined network monitor +2025-01-06 22:27:43,982 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:43,982 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 22:27:43,982 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 3 +2025-01-06 22:27:43,982 DEBUG SenderThread:2124597 [sender.py:send():379] send: stats +2025-01-06 22:27:43,982 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:43,982 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 22:27:43,983 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:43,983 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 22:27:43,983 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 4 +2025-01-06 22:27:43,983 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:43,983 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 22:27:44,818 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:27:45,818 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:50,565 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:27:50,565 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:50,566 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 22:27:50,567 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:50,567 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 22:27:50,567 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 5 +2025-01-06 22:27:50,567 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:27:50,567 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:50,567 DEBUG SenderThread:2124597 [sender.py:send():379] send: history +2025-01-06 22:27:50,567 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 22:27:50,567 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:27:50,569 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:50,569 DEBUG SenderThread:2124597 [sender.py:send():379] send: summary +2025-01-06 22:27:50,569 INFO SenderThread:2124597 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:27:50,570 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:50,570 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 22:27:50,570 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 6 +2025-01-06 22:27:50,570 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:50,570 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 22:27:50,570 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:50,570 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 22:27:50,573 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:27:50,700 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 7 +2025-01-06 22:27:50,700 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:50,700 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 22:27:50,701 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:50,701 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 22:27:50,820 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:27:50,821 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/config.yaml +2025-01-06 22:27:50,927 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:27:52,813 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 8 +2025-01-06 22:27:52,813 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:27:52,813 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:52,813 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 22:27:52,814 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:52,814 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 22:27:52,814 INFO SenderThread:2124597 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 22:27:52,816 INFO SenderThread:2124597 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 22:27:52,818 INFO SenderThread:2124597 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 22:27:52,823 INFO Thread-12 :2124597 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:52,828 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 9 +2025-01-06 22:27:52,828 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:52,828 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 22:27:52,828 DEBUG SenderThread:2124597 [sender.py:send():379] send: artifact +2025-01-06 22:27:52,928 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:27:53,056 INFO SenderThread:2124597 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 22:27:53,056 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:53,056 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 22:27:53,056 INFO SenderThread:2124597 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 22:27:53,824 INFO SenderThread:2124597 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files +2025-01-06 22:27:53,824 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/config.yaml config.yaml +2025-01-06 22:27:53,824 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json wandb-summary.json +2025-01-06 22:27:53,826 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:27:53,827 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/requirements.txt requirements.txt +2025-01-06 22:27:53,827 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-metadata.json wandb-metadata.json +2025-01-06 22:27:53,828 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log output.log +2025-01-06 22:27:53,828 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/diff.patch diff.patch +2025-01-06 22:27:53,828 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 22:27:53,828 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 media/videos/eval/2_eval_2_c776c8b0fd9fc136e372.mp4 +2025-01-06 22:27:53,830 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 media/videos/eval/0_eval_0_925e32212f6150e3263f.mp4 +2025-01-06 22:27:53,830 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 media/videos/eval/1_eval_1_09b93e53fac465967343.mp4 +2025-01-06 22:27:53,831 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 media/videos/eval/4_eval_4_b5c5b2ea7b5d6cafa3af.mp4 +2025-01-06 22:27:53,831 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 media/videos/eval/3_eval_3_f8ac7cd25652f50c9d7e.mp4 +2025-01-06 22:27:53,833 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json media/table/eval/total_success_rate_ci_6_c76fe457b6d22ff6afda.table.json +2025-01-06 22:27:53,833 INFO SenderThread:2124597 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 22:27:53,833 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 10 +2025-01-06 22:27:53,833 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:27:53,833 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:53,833 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 22:27:53,833 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:53,833 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 22:27:53,833 INFO SenderThread:2124597 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:27:53,928 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:27:53,929 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:27:54,013 INFO wandb-upload_1:2124597 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/wandb-summary.json +2025-01-06 22:27:54,032 INFO wandb-upload_3:2124597 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/requirements.txt +2025-01-06 22:27:54,047 INFO wandb-upload_0:2124597 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/config.yaml +2025-01-06 22:27:54,070 INFO wandb-upload_2:2124597 [upload_job.py:push():130] Uploaded file /tmp/tmpgrl4y_r8wandb/xkne6xyz-events.out.tfevents.1736220189.leonmkim-ROG-Strix-G15CS-G15CS.2124431.0 +2025-01-06 22:27:54,078 INFO wandb-upload_4:2124597 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/files/output.log +2025-01-06 22:27:54,279 INFO Thread-11 (_thread_body):2124597 [sender.py:transition_state():614] send defer: 11 +2025-01-06 22:27:54,279 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:54,279 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 22:27:54,279 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:54,279 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 22:27:54,280 INFO SenderThread:2124597 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 22:27:54,281 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 12 +2025-01-06 22:27:54,281 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:54,282 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 22:27:54,282 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:54,282 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 22:27:54,282 INFO SenderThread:2124597 [file_stream.py:finish():601] file stream finish called +2025-01-06 22:27:54,528 INFO SenderThread:2124597 [file_stream.py:finish():605] file stream finish is done +2025-01-06 22:27:54,528 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 13 +2025-01-06 22:27:54,529 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:54,529 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 22:27:54,529 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:54,529 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 22:27:54,529 INFO SenderThread:2124597 [sender.py:transition_state():614] send defer: 14 +2025-01-06 22:27:54,529 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:27:54,529 INFO HandlerThread:2124597 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 22:27:54,529 DEBUG SenderThread:2124597 [sender.py:send():379] send: final +2025-01-06 22:27:54,529 DEBUG SenderThread:2124597 [sender.py:send():379] send: footer +2025-01-06 22:27:54,529 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: defer +2025-01-06 22:27:54,530 INFO SenderThread:2124597 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 22:27:54,530 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:27:54,530 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 22:27:54,530 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:27:54,530 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: server_info +2025-01-06 22:27:54,532 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:27:54,532 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 22:27:54,533 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 22:27:54,533 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:27:54,589 DEBUG SenderThread:2124597 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:27:54,590 INFO MainThread:2124597 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 22:27:54,590 INFO MainThread:2124597 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 22:27:54,590 INFO MainThread:2124597 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 22:27:54,590 DEBUG HandlerThread:2124597 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 22:27:54,591 INFO HandlerThread:2124597 [handler.py:finish():882] shutting down handler +2025-01-06 22:27:55,532 INFO WriterThread:2124597 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/run-or3sumxm.wandb +2025-01-06 22:27:55,590 INFO SenderThread:2124597 [sender.py:finish():1615] shutting down sender +2025-01-06 22:27:55,590 INFO SenderThread:2124597 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:27:55,590 INFO SenderThread:2124597 [file_pusher.py:join():175] waiting for file pusher diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/logs/debug.log b/or3sumxm/wandb/run-20250106_222308-or3sumxm/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..780bd8c9cac4e1fd452cc3d22b4df743b95eb4e5 --- /dev/null +++ b/or3sumxm/wandb/run-20250106_222308-or3sumxm/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Configure stats pid to 2124431 +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/settings +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/logs/debug.log +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/wandb/run-20250106_222308-or3sumxm/logs/debug-internal.log +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_init.py:init():569] calling init triggers +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm'} +2025-01-06 22:23:08,451 INFO MainThread:2124431 [wandb_init.py:init():619] starting backend +2025-01-06 22:23:08,452 INFO MainThread:2124431 [wandb_init.py:init():623] setting up manager +2025-01-06 22:23:08,455 INFO MainThread:2124431 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 22:23:08,456 INFO MainThread:2124431 [wandb_init.py:init():631] backend started and connected +2025-01-06 22:23:08,469 INFO MainThread:2124431 [wandb_init.py:init():720] updated telemetry +2025-01-06 22:23:08,477 INFO MainThread:2124431 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 22:23:08,739 INFO MainThread:2124431 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 22:23:08,807 INFO MainThread:2124431 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 22:23:08,807 INFO MainThread:2124431 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 22:23:09,166 INFO MainThread:2124431 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 22:23:09,167 INFO MainThread:2124431 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 22:23:09,167 INFO MainThread:2124431 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 22:23:09,167 INFO MainThread:2124431 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 22:23:09,168 INFO MainThread:2124431 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 22:23:09,168 INFO MainThread:2124431 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/or3sumxm/tb, True +2025-01-06 22:23:15,534 INFO MainThread:2124431 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 22:27:55,591 WARNING MsgRouterThr:2124431 [router.py:message_loop():77] message_loop has been closed diff --git a/or3sumxm/wandb/run-20250106_222308-or3sumxm/run-or3sumxm.wandb b/or3sumxm/wandb/run-20250106_222308-or3sumxm/run-or3sumxm.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4f258678b25d25a7d7faeb9814345b798d9731ba Binary files /dev/null and b/or3sumxm/wandb/run-20250106_222308-or3sumxm/run-or3sumxm.wandb differ