diff --git a/.gitattributes b/.gitattributes index 59ebb94aaf589a490eea36cd6be91682023234fe..ab40303dd1c08f610e3ca65224e98c347b4953bd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -175,3 +175,31 @@ q6zm6kn3/episode_rosbags/episode_0_2025-01-06-22-52-30.bag filter=lfs diff=lfs m q6zm6kn3/episode_rosbags/episode_1_2025-01-06-22-53-17.bag filter=lfs diff=lfs merge=lfs -text x85ya2ao/episode_rosbags/episode_0_2025-01-07-18-28-53.bag filter=lfs diff=lfs merge=lfs -text m386gyxa/episode_rosbags/episode_2_2025-01-06-20-07-58.bag filter=lfs diff=lfs merge=lfs -text +m386gyxa/episode_rosbags/episode_1_2025-01-06-20-07-09.bag filter=lfs diff=lfs merge=lfs -text +8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 filter=lfs diff=lfs merge=lfs -text +8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 filter=lfs diff=lfs merge=lfs -text +8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 filter=lfs diff=lfs merge=lfs -text +8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 filter=lfs diff=lfs merge=lfs -text +8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 filter=lfs diff=lfs merge=lfs -text +8gt3cc6e/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8gt3cc6e/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8gt3cc6e/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8gt3cc6e/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8gt3cc6e/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +m386gyxa/episode_rosbags/episode_3_2025-01-06-20-08-47.bag filter=lfs diff=lfs merge=lfs -text +up4gjipz/episode_rosbags/episode_0_2025-01-06-20-01-17.bag filter=lfs diff=lfs merge=lfs -text +yjywj48j/episode_rosbags/episode_2_2025-01-07-20-00-16.bag filter=lfs diff=lfs merge=lfs -text +m386gyxa/episode_rosbags/episode_4_2025-01-06-20-09-35.bag filter=lfs diff=lfs merge=lfs -text +yjywj48j/episode_rosbags/episode_3_2025-01-07-20-01-11.bag filter=lfs diff=lfs merge=lfs -text +vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 filter=lfs diff=lfs merge=lfs -text +vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 filter=lfs diff=lfs merge=lfs -text +vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 filter=lfs diff=lfs merge=lfs -text +vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 filter=lfs diff=lfs merge=lfs -text +vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 filter=lfs diff=lfs merge=lfs -text +yjywj48j/episode_rosbags/episode_4_2025-01-07-20-02-04.bag filter=lfs diff=lfs merge=lfs -text +m386gyxa/episode_rosbags/episode_0_2025-01-06-20-06-21.bag filter=lfs diff=lfs merge=lfs -text +vb47oelh/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vb47oelh/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vb47oelh/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vb47oelh/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vb47oelh/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/8gt3cc6e/.hydra/config.yaml b/8gt3cc6e/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd5530cb9a8f25d6d55135607d04c2061b4351d1 --- /dev/null +++ b/8gt3cc6e/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1017_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/8gt3cc6e/.hydra/hydra.yaml b/8gt3cc6e/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf2cc4babdd7f3d5d69b89470552e624ef55d4df --- /dev/null +++ b/8gt3cc6e/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/8gt3cc6e/.hydra/overrides.yaml b/8gt3cc6e/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/8gt3cc6e/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/8gt3cc6e/episode_rosbags/aligned_depth_to_color_K.npy b/8gt3cc6e/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/8gt3cc6e/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/8gt3cc6e/episode_rosbags/color_K.npy b/8gt3cc6e/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/8gt3cc6e/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/8gt3cc6e/eval_video/0_eval.mp4 b/8gt3cc6e/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..320a08ac5fea9eec1d6a5fd22c535ae66f1ed53d --- /dev/null +++ b/8gt3cc6e/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94f52a23cf863a3c113726aef6a114f1f2c6a7d4580d4e826f7ca4bd228bc8ce +size 1411374 diff --git a/8gt3cc6e/eval_video/1_eval.mp4 b/8gt3cc6e/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..109be688fe2378ee2e5bbf31831a5bb189860224 --- /dev/null +++ b/8gt3cc6e/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e6b8d6b47dcf1f2852eb63bdaf21439d1015ea40792816fdd12fe1b84438b76 +size 1408467 diff --git a/8gt3cc6e/eval_video/2_eval.mp4 b/8gt3cc6e/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..848a9b5fe5979c273917ab745e3e7fa206d22c89 --- /dev/null +++ b/8gt3cc6e/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f300fd7a4cb461e89098ec21505e7510d808d825c655c9a880c5a5478d2611e8 +size 1482386 diff --git a/8gt3cc6e/eval_video/3_eval.mp4 b/8gt3cc6e/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c352c70d489cabbacc5d3b22b43bee74af0cfedf --- /dev/null +++ b/8gt3cc6e/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b35454858e15f45db555b98ed83cd632aeb790d3e0a6e0d58f3ebbf52845d267 +size 1568417 diff --git a/8gt3cc6e/eval_video/4_eval.mp4 b/8gt3cc6e/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bfefca13bdc9d34e7b213734be47a3e1f35ce57b --- /dev/null +++ b/8gt3cc6e/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93d560b9b8a81ed37bb536b45665b8a0e311fc13fa6fdedf931a821833ab4d33 +size 1558356 diff --git a/8gt3cc6e/tb/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 b/8gt3cc6e/tb/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 new file mode 100644 index 0000000000000000000000000000000000000000..babdbad384c1fab822879d1176b778840e19bcb0 --- /dev/null +++ b/8gt3cc6e/tb/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6850d126f223ff6ed5f33912b797ed956e75fa9b9302716f5bfc2ba2ca0762f +size 6539 diff --git a/8gt3cc6e/wandb/debug-internal.log b/8gt3cc6e/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e54829409066fd062a371743e73d2d8aa93af64e --- /dev/null +++ b/8gt3cc6e/wandb/debug-internal.log @@ -0,0 +1,871 @@ +2025-01-07 20:05:09,486 INFO StreamThr :2220142 [internal.py:wandb_internal():85] W&B internal server running at pid: 2220142, started at: 2025-01-07 20:05:09.486108 +2025-01-07 20:05:09,487 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:05:09,501 INFO WriterThread:2220142 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/run-8gt3cc6e.wandb +2025-01-07 20:05:09,501 DEBUG SenderThread:2220142 [sender.py:send():379] send: header +2025-01-07 20:05:09,509 DEBUG SenderThread:2220142 [sender.py:send():379] send: run +2025-01-07 20:05:09,796 INFO SenderThread:2220142 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files +2025-01-07 20:05:09,796 INFO SenderThread:2220142 [sender.py:_start_run_threads():1188] run started: 8gt3cc6e with start time 1736298309.486238 +2025-01-07 20:05:09,812 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:05:09,812 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:05:09,887 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:05:09,892 DEBUG HandlerThread:2220142 [system_info.py:__init__():26] System info init +2025-01-07 20:05:09,892 DEBUG HandlerThread:2220142 [system_info.py:__init__():41] System info init done +2025-01-07 20:05:09,892 INFO HandlerThread:2220142 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:05:09,892 INFO SystemMonitor:2220142 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:05:09,892 INFO HandlerThread:2220142 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:05:09,892 INFO SystemMonitor:2220142 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:05:09,893 INFO SystemMonitor:2220142 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:05:09,893 INFO SystemMonitor:2220142 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:05:09,894 INFO SystemMonitor:2220142 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:05:09,894 INFO SystemMonitor:2220142 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:05:09,906 DEBUG HandlerThread:2220142 [system_info.py:probe():152] Probing system +2025-01-07 20:05:09,907 DEBUG HandlerThread:2220142 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:05:09,912 DEBUG HandlerThread:2220142 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:05:09,912 DEBUG HandlerThread:2220142 [system_info.py:probe():200] Probing system done +2025-01-07 20:05:09,912 DEBUG HandlerThread:2220142 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:05:09.906073', 'startedAt': '2025-01-08T01:05:09.471590', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.9895, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.951, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 541.9574737548828}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:05:09,912 INFO HandlerThread:2220142 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:05:09,912 INFO HandlerThread:2220142 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:05:09,912 DEBUG HandlerThread:2220142 [system_info.py:_save_code():44] Saving code +2025-01-07 20:05:09,914 DEBUG HandlerThread:2220142 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:05:09,914 DEBUG HandlerThread:2220142 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:05:09,948 DEBUG HandlerThread:2220142 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:05:09,949 INFO HandlerThread:2220142 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:05:09,952 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:05:09,952 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:05:09,952 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:05:10,196 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:05:10,196 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:05:10,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:05:10,197 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:05:10,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:10,199 INFO HandlerThread:2220142 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/tb" + save: true +} +control { + relay_id: "62165ca8d82b422782b83b5601ae078f" +} +_info { + stream_id: "8gt3cc6e" +} + +2025-01-07 20:05:10,199 DEBUG HandlerThread:2220142 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:05:10,203 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/e7l7e743-wandb-metadata.json +2025-01-07 20:05:10,265 DEBUG SenderThread:2220142 [sender.py:send():379] send: telemetry +2025-01-07 20:05:10,265 DEBUG SenderThread:2220142 [sender.py:send():379] send: tbrecord +2025-01-07 20:05:10,266 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:05:10,266 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 with policy live +2025-01-07 20:05:10,283 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/j9xmsrem-code/FISH/eval_robot.py +2025-01-07 20:05:10,451 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/295elqwg-events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:05:10,798 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:10,798 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-metadata.json +2025-01-07 20:05:10,798 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/code/FISH/eval_robot.py +2025-01-07 20:05:10,798 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/requirements.txt +2025-01-07 20:05:10,799 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:05:10,799 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/code +2025-01-07 20:05:10,799 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/code/FISH +2025-01-07 20:05:11,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:12,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:12,797 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:13,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:14,002 DEBUG SenderThread:2220142 [sender.py:send():379] send: config +2025-01-07 20:05:14,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:15,010 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:15,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:16,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:16,798 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:17,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:18,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:18,799 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:19,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:20,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:20,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:20,800 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:21,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:22,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:22,800 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:23,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:24,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:24,801 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:25,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:05:25,197 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:05:25,237 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:25,316 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:26,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:26,802 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:27,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:28,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:28,802 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:29,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:30,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:31,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:31,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:32,200 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:32,803 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:33,200 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:34,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:35,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:36,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:37,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:37,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:38,203 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:39,203 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:40,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:05:40,197 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:05:40,241 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:41,203 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:42,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:42,269 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:42,806 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/config.yaml +2025-01-07 20:05:43,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:44,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:45,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:46,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:47,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:48,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:48,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:49,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:50,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:51,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:52,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:53,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:54,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:54,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:55,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:05:55,197 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:05:55,245 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:56,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:57,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:58,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:59,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:59,271 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:00,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:01,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:02,206 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:02,698 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:06:02,698 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 with policy now +2025-01-07 20:06:02,699 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:06:02,700 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:06:02,700 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:06:02,700 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:06:02,811 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:02,812 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 +2025-01-07 20:06:02,812 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:06:02,812 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media +2025-01-07 20:06:02,812 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval +2025-01-07 20:06:02,812 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos +2025-01-07 20:06:02,967 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/d2czae8a-media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 +2025-01-07 20:06:03,206 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:03,241 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/fwfcssii-media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 +2025-01-07 20:06:04,207 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:04,710 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:04,812 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:05,207 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:06,210 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:07,207 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:08,207 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:08,813 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:09,207 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:09,894 DEBUG SystemMonitor:2220142 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:06:09,895 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:06:09,895 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:10,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:06:10,197 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:06:10,241 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:10,814 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:11,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:12,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:13,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:14,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:15,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:15,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:16,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:17,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:18,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:19,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:20,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:20,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:21,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:22,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:23,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:24,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:25,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:06:25,197 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:06:25,241 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:25,276 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:26,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:27,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:28,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:29,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:30,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:31,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:31,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:32,210 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:33,210 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:33,822 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:06:34,078 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/8yxk2hzl-events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:06:34,210 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:34,822 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:35,210 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:36,200 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:36,210 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:36,823 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:37,213 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:38,213 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:38,824 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:39,213 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:39,896 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:06:40,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:06:40,198 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:06:40,214 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:40,825 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:41,214 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:41,258 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:42,217 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:43,216 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:44,217 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:44,826 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:45,216 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:46,216 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:47,200 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:47,217 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:48,217 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:49,218 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:50,218 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:51,218 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:52,203 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:52,218 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:53,218 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:54,218 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:55,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:06:55,198 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:06:55,241 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:56,220 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:57,220 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:57,278 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:58,220 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:59,220 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:00,220 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:01,221 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:02,221 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:02,317 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:03,221 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:04,221 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:04,832 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:07:05,224 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:06,223 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:07,223 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:08,200 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:08,223 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:09,223 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:09,896 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:07:10,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:07:10,198 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:07:10,241 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:11,223 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:12,223 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:13,224 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:13,307 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:13,353 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:07:13,354 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:07:13,354 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 with policy now +2025-01-07 20:07:13,354 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:07:13,354 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:07:13,354 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:07:13,631 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/jk6b8tia-media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 +2025-01-07 20:07:13,835 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:07:13,835 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 +2025-01-07 20:07:13,835 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval +2025-01-07 20:07:14,110 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/vrhbh0sd-media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 +2025-01-07 20:07:14,226 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:14,835 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:07:15,230 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:16,227 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:17,232 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:18,229 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:18,837 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:07:18,985 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:19,232 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:20,229 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:20,837 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:07:21,230 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:22,230 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:23,230 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:24,201 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:24,230 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:25,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:07:25,198 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:07:25,241 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:26,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:27,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:28,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:28,640 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:07:28,640 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:07:28,641 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:07:28,840 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:07:28,840 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:07:29,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:29,245 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:30,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:30,841 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:07:31,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:32,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:32,842 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:07:33,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:34,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:34,842 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:07:35,153 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:35,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:36,232 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:37,232 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:38,232 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:39,233 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:39,897 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:07:40,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:07:40,198 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:07:40,245 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:40,267 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:41,234 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:42,234 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:43,234 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:44,234 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:45,234 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:46,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:46,234 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:47,234 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:48,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:49,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:50,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:51,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:51,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:52,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:53,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:54,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:55,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:07:55,198 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:07:55,241 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:56,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:56,282 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:57,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:58,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:59,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:00,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:00,852 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:01,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:02,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:02,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:03,237 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:04,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:05,237 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:06,237 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:07,238 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:07,620 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:08,180 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:08:08,180 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 with policy now +2025-01-07 20:08:08,183 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:08:08,183 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:08:08,183 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:08:08,184 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:08:08,242 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:08,448 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/00boltkl-media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 +2025-01-07 20:08:08,855 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:08,855 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:08:08,855 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 +2025-01-07 20:08:08,856 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval +2025-01-07 20:08:09,165 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/r5tnco6y-media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 +2025-01-07 20:08:09,243 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:09,897 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:08:10,203 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:08:10,203 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:08:10,249 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:10,856 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:11,245 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:12,245 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:13,085 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:13,244 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:14,244 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:14,857 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:15,243 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:16,248 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:16,858 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:17,246 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:18,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:18,246 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:19,249 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:20,254 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:21,252 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:22,249 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:23,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:23,252 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:24,248 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:25,201 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:08:25,201 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:08:25,253 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:26,253 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:27,253 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:28,256 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:28,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:29,257 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:30,255 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:31,255 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:32,255 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:33,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:34,206 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:34,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:35,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:36,263 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:37,266 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:38,262 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:39,242 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:39,259 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:39,504 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:08:39,504 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:08:39,505 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:08:39,865 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:08:39,866 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:08:39,897 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:08:40,151 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/9fcc669c-events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:08:40,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:08:40,204 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:08:40,260 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:40,866 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:41,267 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:42,266 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:42,866 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:43,260 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:44,260 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:44,867 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:45,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:45,260 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:46,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:47,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:48,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:49,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:50,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:50,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:51,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:52,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:53,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:54,262 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:55,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:08:55,202 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:08:55,262 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:55,364 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:56,262 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:57,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:58,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:59,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:00,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:01,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:01,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:02,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:03,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:04,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:05,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:06,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:07,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:07,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:08,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:09,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:09,898 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:09:10,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:09:10,202 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:09:10,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:11,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:12,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:12,353 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:13,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:14,266 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:15,266 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:16,266 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:17,266 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:17,920 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:18,276 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:18,878 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:19,226 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:09:19,226 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 with policy now +2025-01-07 20:09:19,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:09:19,236 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:09:19,236 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:09:19,243 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:09:19,270 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:19,501 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/6lyono38-media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 +2025-01-07 20:09:19,879 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:09:19,879 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 +2025-01-07 20:09:19,880 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval +2025-01-07 20:09:20,174 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/xts3snmt-media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 +2025-01-07 20:09:20,285 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:20,880 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:21,281 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:22,281 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:22,974 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:23,280 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:24,280 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:24,881 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:25,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:09:25,205 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:09:25,280 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:26,279 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:26,882 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:27,282 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:28,283 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:28,817 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:29,285 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:30,288 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:31,284 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:32,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:33,285 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:34,215 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:34,286 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:35,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:36,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:37,288 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:37,885 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:09:38,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:38,578 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:09:38,578 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:09:38,578 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:09:38,885 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:38,886 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:09:39,293 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:39,805 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:39,898 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:09:40,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:09:40,205 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:09:40,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:40,886 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:41,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:42,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:42,887 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:43,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:44,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:45,213 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:45,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:46,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:47,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:48,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:49,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:50,214 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:50,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:51,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:52,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:53,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:54,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:55,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:09:55,204 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:09:55,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:55,333 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:56,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:57,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:58,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:58,892 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:59,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:00,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:00,472 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:01,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:02,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:02,894 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:10:03,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:04,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:05,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:06,214 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:06,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:06,895 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:10:07,292 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:08,292 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:09,292 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:09,899 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:10:10,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:10:10,204 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:10:10,292 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:11,292 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:11,310 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:12,292 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:13,293 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:14,293 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:15,293 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:16,293 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:16,313 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:17,300 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:17,614 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:10:17,614 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 with policy now +2025-01-07 20:10:17,616 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:10:17,617 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:10:17,617 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:17,627 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:17,882 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/0c7ap176-media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 +2025-01-07 20:10:17,899 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:10:17,899 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 +2025-01-07 20:10:17,899 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval +2025-01-07 20:10:18,191 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/08idz8if-media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 +2025-01-07 20:10:18,311 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:18,899 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:10:19,302 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:20,308 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:21,309 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:21,381 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:22,304 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:22,900 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:10:23,310 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:24,314 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:25,210 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:10:25,210 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:10:25,309 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:26,317 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:26,902 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:10:27,312 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:27,369 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:28,311 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:29,314 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:30,325 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:31,329 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:32,326 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:33,227 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:33,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:34,327 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:35,324 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:36,345 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:36,648 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:10:36,648 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:36,649 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:36,756 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:10:36,756 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:10:36,905 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:10:36,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:10:36,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:10:36,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media +2025-01-07 20:10:36,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table +2025-01-07 20:10:36,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval +2025-01-07 20:10:36,950 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:10:37,145 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/u8grp3lf-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:10:37,170 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/lk3h76u6-events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:10:37,266 INFO wandb-upload_2:2220142 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzcs05d44 +2025-01-07 20:10:37,321 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:37,375 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:10:37,421 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:10:37,705 INFO SenderThread:2220142 [sender.py:send_request_log_artifact():1525] logged artifact run-8gt3cc6e-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM1ODYzNg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgyNDY0MQ==', 'latestArtifact': None}} +2025-01-07 20:10:37,705 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:10:37,705 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 20:10:37,705 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:10:37,705 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,706 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,706 DEBUG SenderThread:2220142 [sender.py:send():379] send: telemetry +2025-01-07 20:10:37,706 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,710 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,710 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,710 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,710 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,711 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,711 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,711 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,711 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,712 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,712 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,712 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,712 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,713 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,713 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,713 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,713 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,714 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,714 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,714 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,714 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,715 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,715 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:10:37,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:10:37,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:10:37,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json +2025-01-07 20:10:37,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval +2025-01-07 20:10:37,969 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/8j0kqxm8-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:10:38,088 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/4n1dp5b1-media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json +2025-01-07 20:10:38,093 INFO wandb-upload_2:2220142 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppc617cxg +2025-01-07 20:10:38,321 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:38,538 INFO SenderThread:2220142 [sender.py:send_request_log_artifact():1525] logged artifact run-8gt3cc6e-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM1ODY1OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgyNDY0NQ==', 'latestArtifact': None}} +2025-01-07 20:10:38,539 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:10:38,539 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json with policy now +2025-01-07 20:10:38,539 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:10:38,539 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:38,540 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:38,540 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:38,830 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/2wg74seq-media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json +2025-01-07 20:10:38,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:10:38,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:10:39,321 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:39,899 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:10:40,206 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:10:40,206 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:10:40,321 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:41,321 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:42,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:43,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:44,305 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:44,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:45,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:46,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:47,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:48,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:49,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:50,220 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:50,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:51,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:52,323 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:52,437 DEBUG SenderThread:2220142 [sender.py:send():379] send: exit +2025-01-07 20:10:52,438 INFO SenderThread:2220142 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:10:52,438 INFO SenderThread:2220142 [sender.py:send_exit():588] handling runtime: 342 +2025-01-07 20:10:52,438 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:52,438 INFO SenderThread:2220142 [sender.py:send_exit():594] send defer +2025-01-07 20:10:52,438 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:52,438 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:10:52,439 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:52,439 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:10:52,439 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:10:52,439 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:52,439 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:10:52,439 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:52,439 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:10:52,439 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:10:52,439 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:52,439 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:10:52,439 INFO HandlerThread:2220142 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:10:52,439 INFO HandlerThread:2220142 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:10:52,440 INFO HandlerThread:2220142 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:10:52,440 DEBUG SystemMonitor:2220142 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:10:52,440 DEBUG SystemMonitor:2220142 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:10:52,494 INFO HandlerThread:2220142 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:10:52,494 INFO HandlerThread:2220142 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:10:52,494 INFO HandlerThread:2220142 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:10:52,494 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:52,494 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:10:52,494 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:10:52,494 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:10:52,495 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:52,495 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:10:52,495 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:52,495 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:10:52,495 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:10:52,495 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:52,495 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:10:52,911 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:10:54,912 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:10:58,656 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:10:58,656 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:58,656 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:58,656 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:10:58,656 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:10:58,657 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:10:58,658 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:10:58,658 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:10:58,658 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:58,658 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:58,658 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:10:58,659 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:58,659 DEBUG SenderThread:2220142 [sender.py:send():379] send: summary +2025-01-07 20:10:58,663 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:58,663 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:58,663 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:10:58,663 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:10:58,663 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:58,663 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:10:58,663 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:58,663 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:10:58,667 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:58,807 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:10:58,807 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:58,807 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:10:58,807 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:58,807 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:10:58,914 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/config.yaml +2025-01-07 20:10:58,914 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:10:59,439 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:11:02,357 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:11:02,357 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:11:02,357 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:11:02,358 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:11:02,358 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:11:02,358 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:11:02,358 INFO SenderThread:2220142 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:11:02,360 INFO SenderThread:2220142 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:11:02,362 INFO SenderThread:2220142 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:11:02,374 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:11:02,374 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:11:02,374 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:11:02,374 DEBUG SenderThread:2220142 [sender.py:send():379] send: artifact +2025-01-07 20:11:02,440 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:11:02,558 INFO SenderThread:2220142 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc1Nzg0Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:11:02,559 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:11:02,559 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:11:02,559 INFO SenderThread:2220142 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:11:02,915 INFO SenderThread:2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:11:02,915 INFO SenderThread:2220142 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files +2025-01-07 20:11:02,915 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/config.yaml config.yaml +2025-01-07 20:11:02,916 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json wandb-summary.json +2025-01-07 20:11:02,916 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/requirements.txt requirements.txt +2025-01-07 20:11:02,916 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:11:02,916 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:11:02,916 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log output.log +2025-01-07 20:11:02,919 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:11:02,921 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 +2025-01-07 20:11:02,921 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 +2025-01-07 20:11:02,921 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 +2025-01-07 20:11:02,922 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 +2025-01-07 20:11:02,922 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 +2025-01-07 20:11:02,923 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json +2025-01-07 20:11:02,923 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:11:02,923 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:11:02,924 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:11:02,925 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:11:02,925 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:11:02,925 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:11:02,925 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:11:02,925 INFO SenderThread:2220142 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:11:03,108 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:11:03,117 INFO wandb-upload_2:2220142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/config.yaml +2025-01-07 20:11:03,119 INFO wandb-upload_4:2220142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:11:03,126 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/requirements.txt +2025-01-07 20:11:03,224 INFO wandb-upload_3:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/oc0glxge-events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:11:03,424 INFO Thread-11 (_thread_body):2220142 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:11:03,424 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:11:03,424 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:11:03,424 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:11:03,425 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:11:03,425 INFO SenderThread:2220142 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:11:03,426 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:11:03,426 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:11:03,426 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:11:03,426 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:11:03,426 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:11:03,426 INFO SenderThread:2220142 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:11:03,440 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:11:03,649 INFO SenderThread:2220142 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:11:03,650 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:11:03,650 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:11:03,650 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:11:03,650 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:11:03,650 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:11:03,650 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:11:03,650 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:11:03,650 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:11:03,650 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:11:03,651 DEBUG SenderThread:2220142 [sender.py:send():379] send: final +2025-01-07 20:11:03,651 DEBUG SenderThread:2220142 [sender.py:send():379] send: footer +2025-01-07 20:11:03,651 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:11:03,651 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:11:03,651 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:11:03,651 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:11:03,652 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:11:03,652 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:11:03,652 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:11:03,652 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:11:03,652 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:11:03,654 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:03,703 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:11:03,704 INFO MainThread:2220142 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:11:03,704 INFO MainThread:2220142 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:11:03,704 INFO MainThread:2220142 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:11:03,704 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:11:03,705 INFO HandlerThread:2220142 [handler.py:finish():882] shutting down handler +2025-01-07 20:11:04,652 INFO WriterThread:2220142 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/run-8gt3cc6e.wandb +2025-01-07 20:11:04,704 INFO SenderThread:2220142 [sender.py:finish():1615] shutting down sender +2025-01-07 20:11:04,704 INFO SenderThread:2220142 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:11:04,704 INFO SenderThread:2220142 [file_pusher.py:join():175] waiting for file pusher diff --git a/8gt3cc6e/wandb/debug.log b/8gt3cc6e/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a96f32c1f152a3339eae44fc3cc183d5740d6870 --- /dev/null +++ b/8gt3cc6e/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Configure stats pid to 2219968 +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/settings +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:05:09,479 INFO MainThread:2219968 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/logs/debug.log +2025-01-07 20:05:09,479 INFO MainThread:2219968 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/logs/debug-internal.log +2025-01-07 20:05:09,479 INFO MainThread:2219968 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:05:09,479 INFO MainThread:2219968 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e'} +2025-01-07 20:05:09,479 INFO MainThread:2219968 [wandb_init.py:init():619] starting backend +2025-01-07 20:05:09,479 INFO MainThread:2219968 [wandb_init.py:init():623] setting up manager +2025-01-07 20:05:09,485 INFO MainThread:2219968 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:05:09,486 INFO MainThread:2219968 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:05:09,500 INFO MainThread:2219968 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:05:09,507 INFO MainThread:2219968 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:05:09,811 INFO MainThread:2219968 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:05:09,878 INFO MainThread:2219968 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:05:09,878 INFO MainThread:2219968 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:05:10,197 INFO MainThread:2219968 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:05:10,197 INFO MainThread:2219968 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:05:10,197 INFO MainThread:2219968 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:05:10,197 INFO MainThread:2219968 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:05:10,199 INFO MainThread:2219968 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:05:10,199 INFO MainThread:2219968 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/tb, True +2025-01-07 20:05:14,001 INFO MainThread:2219968 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:11:04,705 WARNING MsgRouterThr:2219968 [router.py:message_loop():77] message_loop has been closed diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/code/FISH/eval_robot.py b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/config.yaml b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5fe1f41339c9af76071c58958d2bec0c07dbe772 --- /dev/null +++ b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1017_0' +wandb_notes: + desc: null + value: 1017_0_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736298309 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json new file mode 100644 index 0000000000000000000000000000000000000000..68c919984eaead3470184d43136c584665a3715e --- /dev/null +++ b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[37, 55, 0.6727272727272727, 0.5135522484752251, 0.8063256888969028, 0.5449897795825625, 0.7924889176498991, 0.5658669024574152, 0.7668133526506855, 0.5821256674661894, 0.7550052180376259]]} \ No newline at end of file diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..320a08ac5fea9eec1d6a5fd22c535ae66f1ed53d --- /dev/null +++ b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94f52a23cf863a3c113726aef6a114f1f2c6a7d4580d4e826f7ca4bd228bc8ce +size 1411374 diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..109be688fe2378ee2e5bbf31831a5bb189860224 --- /dev/null +++ b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e6b8d6b47dcf1f2852eb63bdaf21439d1015ea40792816fdd12fe1b84438b76 +size 1408467 diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..848a9b5fe5979c273917ab745e3e7fa206d22c89 --- /dev/null +++ b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f300fd7a4cb461e89098ec21505e7510d808d825c655c9a880c5a5478d2611e8 +size 1482386 diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c352c70d489cabbacc5d3b22b43bee74af0cfedf --- /dev/null +++ b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b35454858e15f45db555b98ed83cd632aeb790d3e0a6e0d58f3ebbf52845d267 +size 1568417 diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bfefca13bdc9d34e7b213734be47a3e1f35ce57b --- /dev/null +++ b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93d560b9b8a81ed37bb536b45665b8a0e311fc13fa6fdedf931a821833ab4d33 +size 1558356 diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b57948393f3c663e4b066f83138a40cd0a3a95cd --- /dev/null +++ b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log @@ -0,0 +1,270 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736298314.002080]: resetting environment +[INFO] [1736298314.006665]: cleared current plan +[INFO] [1736298314.007480]: moving to home +[INFO] [1736298315.009291]: reached home +[INFO] [1736298315.012012]: reset action history +[INFO] [1736298317.720162]: environment reset +Starting episode 0 +[INFO] [1736298317.724546]: resetting environment +[INFO] [1736298317.729124]: cleared current plan +[INFO] [1736298317.730147]: moving to home +[INFO] [1736298318.731739]: reached home +[INFO] [1736298318.732922]: reset action history +[INFO] [1736298321.449742]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736298323.105819]: resetting environment +[INFO] [1736298323.111817]: cleared current plan +[INFO] [1736298323.113892]: moving to home +[INFO] [1736298324.115984]: reached home +[INFO] [1736298324.117255]: reset action history +[INFO] [1736298326.826466]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/episode_rosbags/episode_0_2025-01-07-20-05-26.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2219968.log +started roslaunch server http://158.130.50.37:46089/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2220257] +process[print_text-2]: started with pid [2220258] +process[rosbag_record-3]: started with pid [2220259] +started bagging! +For topic gripper_width: timestamp difference is -146644639 for nearest: 1736298331502682639 - target: 1736298331649327278 at idx 58 +[INFO] [1736298361.689879]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736298362.485245]: Stored episode 1. +[INFO] [1736298362.485912]: Saving video... +[INFO] [1736298362.686558]: Video saved! +Starting episode 1 +[INFO] [1736298362.699967]: resetting environment +[INFO] [1736298362.707913]: cleared current plan +[INFO] [1736298362.709054]: moving to home +[INFO] [1736298366.312185]: reached home +[INFO] [1736298366.316210]: reset action history +[INFO] [1736298369.025676]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736298393.316452]: resetting environment +[INFO] [1736298393.322877]: cleared current plan +[INFO] [1736298393.323074]: moving to home +[INFO] [1736298394.324711]: reached home +[INFO] [1736298394.325110]: reset action history +[INFO] [1736298397.033442]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/episode_rosbags/episode_1_2025-01-07-20-06-37.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2219968.log +started roslaunch server http://158.130.50.37:35241/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2220394] +process[print_text-5]: started with pid [2220395] +process[rosbag_record-6]: started with pid [2220396] +started bagging! +For topic gripper_width: timestamp difference is 107620904 for nearest: 1736298399069348046 - target: 1736298398961727142 at idx 30 +For topic gripper_width: timestamp difference is 108292212 for nearest: 1736298399869363417 - target: 1736298399761071205 at idx 37 +For topic gripper_width: timestamp difference is 111403275 for nearest: 1736298403769354153 - target: 1736298403657950878 at idx 93 +For topic gripper_width: timestamp difference is 126331479 for nearest: 1736298422269355924 - target: 1736298422143024445 at idx 249 +[INFO] [1736298432.017490]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736298432.795085]: Stored episode 2. +[INFO] [1736298432.796373]: Saving video... +[INFO] [1736298433.306744]: Video saved! +Starting episode 2 +[INFO] [1736298433.327167]: resetting environment +[INFO] [1736298433.336346]: cleared current plan +[INFO] [1736298433.336566]: moving to home +[INFO] [1736298436.938744]: reached home +[INFO] [1736298436.947356]: reset action history +[INFO] [1736298439.722162]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736298448.191871]: resetting environment +[INFO] [1736298448.202498]: cleared current plan +[INFO] [1736298448.202764]: moving to home +[INFO] [1736298449.204471]: reached home +[INFO] [1736298449.204903]: reset action history +[INFO] [1736298451.914179]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/episode_rosbags/episode_2_2025-01-07-20-07-31.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2219968.log +started roslaunch server http://158.130.50.37:41833/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2220537] +process[print_text-8]: started with pid [2220538] +process[rosbag_record-9]: started with pid [2220539] +started bagging! +For topic gripper_width: timestamp difference is 109306404 for nearest: 1736298478969377586 - target: 1736298478860071182 at idx 249 +[INFO] [1736298486.824528]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1736298487.618630]: Stored episode 3. +[INFO] [1736298487.621549]: Saving video... +[WARN] [1736298488.189882]: Plan exhausted +[WARN] [1736298488.230140]: Plan exhausted +[INFO] [1736298488.160600]: Video saved! +Starting episode 3 +[INFO] [1736298488.197175]: resetting environment +[INFO] [1736298488.235538]: cleared current plan +[INFO] [1736298488.235809]: moving to home +[INFO] [1736298492.041034]: reached home +[INFO] [1736298492.051775]: reset action history +[INFO] [1736298494.827299]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736298519.231686]: resetting environment +[INFO] [1736298519.256414]: cleared current plan +[INFO] [1736298519.257574]: moving to home +[INFO] [1736298520.260158]: reached home +[INFO] [1736298520.270248]: reset action history +[INFO] [1736298522.995107]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/episode_rosbags/episode_3_2025-01-07-20-08-43.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2219968.log +started roslaunch server http://158.130.50.37:34879/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2220697] +process[print_text-11]: started with pid [2220698] +process[rosbag_record-12]: started with pid [2220699] +started bagging! +[INFO] [1736298557.919965]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1736298558.682728]: Stored episode 4. +[INFO] [1736298558.683335]: Saving video... +[INFO] [1736298559.176229]: Video saved! +Starting episode 4 +[INFO] [1736298559.241808]: resetting environment +[INFO] [1736298559.259783]: cleared current plan +[INFO] [1736298559.260033]: moving to home +[INFO] [1736298562.963566]: reached home +[INFO] [1736298562.976712]: reset action history +[INFO] [1736298565.768080]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736298577.724326]: resetting environment +[INFO] [1736298577.750460]: cleared current plan +[INFO] [1736298577.750728]: moving to home +[INFO] [1736298578.756144]: reached home +[INFO] [1736298578.765556]: reset action history +[INFO] [1736298581.493085]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/episode_rosbags/episode_4_2025-01-07-20-09-41.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2219968.log +started roslaunch server http://158.130.50.37:35607/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2220838] +process[print_text-14]: started with pid [2220839] +process[rosbag_record-15]: started with pid [2220840] +started bagging! +For topic gripper_width: timestamp difference is 100034622 for nearest: 1736298596302680162 - target: 1736298596202645540 at idx 204 +For topic gripper_width: timestamp difference is 103486654 for nearest: 1736298600469355699 - target: 1736298600365869045 at idx 249 +For topic gripper_width: timestamp difference is 106950587 for nearest: 1736298604669354504 - target: 1736298604562403917 at idx 249 +[INFO] [1736298616.312908]: Storing episode... +[pub_text-13] killing on exit[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[INFO] [1736298617.092862]: Stored episode 5. +[INFO] [1736298617.102173]: Saving video... +[INFO] [1736298617.561806]: Video saved! +[INFO] [1736298617.617690]: resetting environment +[INFO] [1736298617.663656]: cleared current plan +[INFO] [1736298617.670744]: moving to home +[INFO] [1736298621.380301]: reached home +[INFO] [1736298621.400793]: reset action history +[INFO] [1736298624.195692]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/requirements.txt b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-metadata.json b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e36055c1b5f1457e9c7dd653a92938e2679690bd --- /dev/null +++ b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-08T01:05:09.906073", + "startedAt": "2025-01-08T01:05:09.471590", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.9895, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.951, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 541.9574737548828 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9b158abf527b951cb7252d3ce75481cea42b9b00 --- /dev/null +++ b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "94f52a23cf863a3c113726aef6a114f1f2c6a7d4580d4e826f7ca4bd228bc8ce", "size": 1411374, "path": "media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4"}, "global_step": 665, "_timestamp": 1736298637.2875807, "_runtime": 327.8013427257538, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "4e6b8d6b47dcf1f2852eb63bdaf21439d1015ea40792816fdd12fe1b84438b76", "size": 1408467, "path": "media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4"}, "eval/num_success": 0.0, "episode": 55.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "f300fd7a4cb461e89098ec21505e7510d808d825c655c9a880c5a5478d2611e8", "size": 1482386, "path": "media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "b35454858e15f45db555b98ed83cd632aeb790d3e0a6e0d58f3ebbf52845d267", "size": 1568417, "path": "media/videos/eval/3_eval_3_b35454858e15f45db555.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "93d560b9b8a81ed37bb536b45665b8a0e311fc13fa6fdedf931a821833ab4d33", "size": 1558356, "path": "media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://4pqoui3roo7qcf6hrr9yqmzotrrsumojrk4hftlkjbpa5vpcjq6m6kv4cqrl49nrkqigdlbwsewq09zfulv8w0wy3t2ro39z4xkw7i9idjtujfsowidvllzvckc9sa8y/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://qz9skqz5w2mk6cwyzb991ruxak4ux4ge67vev9ew9ej34bzx8fun0tf7niwozt1nb7x32q7wmf5npy8c01jlssc2uqvum0uqeg2eakylzkt6j37m8pgkhl4grlk3z1jl:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 37, "total_num_episodes": 55, "total_success_rate": 0.6727272727272727, "total_success_rate_lb_0.01": 0.5135522484752251, "total_success_rate_ub_0.01": 0.8063256888969028, "total_success_rate_lb_0.025": 0.5449897795825625, "total_success_rate_ub_0.025": 0.7924889176498991, "total_success_rate_lb_0.05": 0.5658669024574152, "total_success_rate_ub_0.05": 0.7668133526506855, "total_success_rate_lb_0.1": 0.5821256674661894, "total_success_rate_ub_0.1": 0.7550052180376259, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "130e835d6612d43dcccddeca178b98a0c7a113665ffa2ae7bf1f5795ff4616e0", "size": 520, "artifact_path": "wandb-client-artifact://xhb2wvuje9botu6r25dfe98g0h6nff60qyp876f6aezj06vvedblunfq3qynrvpzii942wcpqvym20xn7aq7u3fqj5wsovrfvwl71sp9ejfy4dagkg54285axr5p1uua/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://zib78gqk0fh5jlgqo2m63umclokm1pezcccrb5pw4sugo71b9bqm1o12j4lkxrex4gsft8wvjnsvhukouz8cx55r14yamrvob8nakykjkukjb0rvo1qq3jzphou012mj:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 342}, "eval/total_success_rate": 0.6727272868156433, "eval/total_success_rate_lb_0.01": 0.5135522484779358, "eval/total_success_rate_ub_0.01": 0.8063256740570068, "eval/total_success_rate_lb_0.025": 0.5449897646903992, "eval/total_success_rate_ub_0.025": 0.7924889326095581, "eval/total_success_rate_lb_0.05": 0.5658668875694275, "eval/total_success_rate_ub_0.05": 0.766813337802887, "eval/total_success_rate_lb_0.1": 0.5821256637573242, "eval/total_success_rate_ub_0.1": 0.7550052404403687} \ No newline at end of file diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/logs/debug-internal.log b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e54829409066fd062a371743e73d2d8aa93af64e --- /dev/null +++ b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/logs/debug-internal.log @@ -0,0 +1,871 @@ +2025-01-07 20:05:09,486 INFO StreamThr :2220142 [internal.py:wandb_internal():85] W&B internal server running at pid: 2220142, started at: 2025-01-07 20:05:09.486108 +2025-01-07 20:05:09,487 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:05:09,501 INFO WriterThread:2220142 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/run-8gt3cc6e.wandb +2025-01-07 20:05:09,501 DEBUG SenderThread:2220142 [sender.py:send():379] send: header +2025-01-07 20:05:09,509 DEBUG SenderThread:2220142 [sender.py:send():379] send: run +2025-01-07 20:05:09,796 INFO SenderThread:2220142 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files +2025-01-07 20:05:09,796 INFO SenderThread:2220142 [sender.py:_start_run_threads():1188] run started: 8gt3cc6e with start time 1736298309.486238 +2025-01-07 20:05:09,812 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:05:09,812 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:05:09,887 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:05:09,892 DEBUG HandlerThread:2220142 [system_info.py:__init__():26] System info init +2025-01-07 20:05:09,892 DEBUG HandlerThread:2220142 [system_info.py:__init__():41] System info init done +2025-01-07 20:05:09,892 INFO HandlerThread:2220142 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:05:09,892 INFO SystemMonitor:2220142 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:05:09,892 INFO HandlerThread:2220142 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:05:09,892 INFO SystemMonitor:2220142 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:05:09,893 INFO SystemMonitor:2220142 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:05:09,893 INFO SystemMonitor:2220142 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:05:09,894 INFO SystemMonitor:2220142 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:05:09,894 INFO SystemMonitor:2220142 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:05:09,906 DEBUG HandlerThread:2220142 [system_info.py:probe():152] Probing system +2025-01-07 20:05:09,907 DEBUG HandlerThread:2220142 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:05:09,912 DEBUG HandlerThread:2220142 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:05:09,912 DEBUG HandlerThread:2220142 [system_info.py:probe():200] Probing system done +2025-01-07 20:05:09,912 DEBUG HandlerThread:2220142 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:05:09.906073', 'startedAt': '2025-01-08T01:05:09.471590', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.9895, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.951, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 541.9574737548828}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:05:09,912 INFO HandlerThread:2220142 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:05:09,912 INFO HandlerThread:2220142 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:05:09,912 DEBUG HandlerThread:2220142 [system_info.py:_save_code():44] Saving code +2025-01-07 20:05:09,914 DEBUG HandlerThread:2220142 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:05:09,914 DEBUG HandlerThread:2220142 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:05:09,948 DEBUG HandlerThread:2220142 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:05:09,949 INFO HandlerThread:2220142 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:05:09,952 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:05:09,952 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:05:09,952 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:05:10,196 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:05:10,196 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:05:10,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:05:10,197 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:05:10,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:10,199 INFO HandlerThread:2220142 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/tb" + save: true +} +control { + relay_id: "62165ca8d82b422782b83b5601ae078f" +} +_info { + stream_id: "8gt3cc6e" +} + +2025-01-07 20:05:10,199 DEBUG HandlerThread:2220142 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:05:10,203 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/e7l7e743-wandb-metadata.json +2025-01-07 20:05:10,265 DEBUG SenderThread:2220142 [sender.py:send():379] send: telemetry +2025-01-07 20:05:10,265 DEBUG SenderThread:2220142 [sender.py:send():379] send: tbrecord +2025-01-07 20:05:10,266 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:05:10,266 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 with policy live +2025-01-07 20:05:10,283 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/j9xmsrem-code/FISH/eval_robot.py +2025-01-07 20:05:10,451 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/295elqwg-events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:05:10,798 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:10,798 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-metadata.json +2025-01-07 20:05:10,798 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/code/FISH/eval_robot.py +2025-01-07 20:05:10,798 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/requirements.txt +2025-01-07 20:05:10,799 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:05:10,799 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/code +2025-01-07 20:05:10,799 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/code/FISH +2025-01-07 20:05:11,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:12,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:12,797 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:13,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:14,002 DEBUG SenderThread:2220142 [sender.py:send():379] send: config +2025-01-07 20:05:14,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:15,010 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:15,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:16,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:16,798 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:17,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:18,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:18,799 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:19,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:20,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:20,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:20,800 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:21,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:22,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:22,800 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:23,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:24,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:24,801 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:25,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:05:25,197 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:05:25,237 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:25,316 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:26,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:26,802 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:27,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:28,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:28,802 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:29,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:30,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:31,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:31,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:32,200 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:32,803 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:05:33,200 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:34,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:35,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:36,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:37,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:37,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:38,203 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:39,203 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:40,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:05:40,197 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:05:40,241 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:41,203 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:42,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:42,269 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:42,806 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/config.yaml +2025-01-07 20:05:43,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:44,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:45,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:46,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:47,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:48,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:48,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:49,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:50,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:51,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:52,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:53,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:54,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:05:54,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:55,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:05:55,197 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:05:55,245 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:56,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:57,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:58,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:59,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:05:59,271 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:00,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:01,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:02,206 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:02,698 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:06:02,698 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 with policy now +2025-01-07 20:06:02,699 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:06:02,700 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:06:02,700 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:06:02,700 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:06:02,811 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:02,812 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 +2025-01-07 20:06:02,812 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:06:02,812 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media +2025-01-07 20:06:02,812 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval +2025-01-07 20:06:02,812 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos +2025-01-07 20:06:02,967 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/d2czae8a-media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 +2025-01-07 20:06:03,206 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:03,241 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/fwfcssii-media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 +2025-01-07 20:06:04,207 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:04,710 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:04,812 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:05,207 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:06,210 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:07,207 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:08,207 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:08,813 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:09,207 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:09,894 DEBUG SystemMonitor:2220142 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:06:09,895 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:06:09,895 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:10,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:06:10,197 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:06:10,241 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:10,814 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:11,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:12,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:13,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:14,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:15,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:15,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:16,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:17,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:18,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:19,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:20,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:20,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:21,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:22,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:23,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:24,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:25,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:06:25,197 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:06:25,241 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:25,276 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:26,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:27,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:28,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:29,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:30,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:31,199 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:31,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:32,210 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:33,210 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:33,822 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:06:34,078 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/8yxk2hzl-events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:06:34,210 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:34,822 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:35,210 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:36,200 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:36,210 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:36,823 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:37,213 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:38,213 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:38,824 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:39,213 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:39,896 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:06:40,197 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:06:40,198 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:06:40,214 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:40,825 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:41,214 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:41,258 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:42,217 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:43,216 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:44,217 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:44,826 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:06:45,216 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:46,216 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:47,200 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:47,217 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:48,217 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:49,218 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:50,218 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:51,218 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:52,203 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:52,218 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:53,218 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:54,218 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:55,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:06:55,198 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:06:55,241 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:56,220 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:57,220 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:57,278 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:06:58,220 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:06:59,220 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:00,220 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:01,221 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:02,221 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:02,317 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:03,221 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:04,221 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:04,832 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:07:05,224 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:06,223 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:07,223 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:08,200 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:08,223 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:09,223 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:09,896 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:07:10,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:07:10,198 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:07:10,241 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:11,223 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:12,223 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:13,224 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:13,307 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:13,353 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:07:13,354 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:07:13,354 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 with policy now +2025-01-07 20:07:13,354 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:07:13,354 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:07:13,354 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:07:13,631 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/jk6b8tia-media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 +2025-01-07 20:07:13,835 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:07:13,835 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 +2025-01-07 20:07:13,835 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval +2025-01-07 20:07:14,110 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/vrhbh0sd-media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 +2025-01-07 20:07:14,226 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:14,835 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:07:15,230 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:16,227 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:17,232 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:18,229 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:18,837 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:07:18,985 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:19,232 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:20,229 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:20,837 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:07:21,230 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:22,230 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:23,230 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:24,201 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:24,230 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:25,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:07:25,198 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:07:25,241 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:26,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:27,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:28,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:28,640 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:07:28,640 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:07:28,641 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:07:28,840 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:07:28,840 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:07:29,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:29,245 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:30,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:30,841 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:07:31,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:32,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:32,842 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:07:33,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:34,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:34,842 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:07:35,153 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:35,231 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:36,232 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:37,232 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:38,232 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:39,233 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:39,897 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:07:40,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:07:40,198 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:07:40,245 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:40,267 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:41,234 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:42,234 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:43,234 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:44,234 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:45,234 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:46,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:46,234 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:47,234 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:48,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:49,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:50,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:51,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:51,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:52,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:53,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:54,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:55,198 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:07:55,198 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:07:55,241 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:56,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:56,282 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:07:57,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:58,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:07:59,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:00,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:00,852 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:01,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:02,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:02,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:03,237 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:04,236 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:05,237 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:06,237 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:07,238 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:07,620 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:08,180 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:08:08,180 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 with policy now +2025-01-07 20:08:08,183 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:08:08,183 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:08:08,183 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:08:08,184 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:08:08,242 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:08,448 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/00boltkl-media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 +2025-01-07 20:08:08,855 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:08,855 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:08:08,855 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 +2025-01-07 20:08:08,856 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval +2025-01-07 20:08:09,165 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/r5tnco6y-media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 +2025-01-07 20:08:09,243 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:09,897 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:08:10,203 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:08:10,203 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:08:10,249 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:10,856 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:11,245 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:12,245 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:13,085 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:13,244 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:14,244 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:14,857 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:15,243 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:16,248 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:16,858 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:17,246 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:18,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:18,246 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:19,249 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:20,254 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:21,252 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:22,249 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:23,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:23,252 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:24,248 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:25,201 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:08:25,201 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:08:25,253 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:26,253 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:27,253 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:28,256 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:28,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:29,257 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:30,255 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:31,255 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:32,255 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:33,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:34,206 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:34,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:35,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:36,263 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:37,266 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:38,262 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:39,242 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:39,259 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:39,504 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:08:39,504 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:08:39,505 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:08:39,865 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:08:39,866 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:08:39,897 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:08:40,151 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/9fcc669c-events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:08:40,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:08:40,204 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:08:40,260 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:40,866 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:41,267 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:42,266 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:42,866 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:43,260 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:44,260 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:44,867 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:08:45,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:45,260 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:46,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:47,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:48,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:49,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:50,208 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:50,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:51,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:52,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:53,261 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:54,262 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:55,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:08:55,202 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:08:55,262 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:55,364 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:08:56,262 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:57,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:58,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:08:59,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:00,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:01,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:01,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:02,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:03,264 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:04,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:05,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:06,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:07,209 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:07,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:08,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:09,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:09,898 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:09:10,202 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:09:10,202 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:09:10,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:11,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:12,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:12,353 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:13,265 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:14,266 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:15,266 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:16,266 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:17,266 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:17,920 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:18,276 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:18,878 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:19,226 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:09:19,226 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 with policy now +2025-01-07 20:09:19,235 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:09:19,236 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:09:19,236 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:09:19,243 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:09:19,270 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:19,501 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/6lyono38-media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 +2025-01-07 20:09:19,879 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:09:19,879 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 +2025-01-07 20:09:19,880 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval +2025-01-07 20:09:20,174 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/xts3snmt-media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 +2025-01-07 20:09:20,285 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:20,880 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:21,281 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:22,281 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:22,974 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:23,280 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:24,280 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:24,881 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:25,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:09:25,205 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:09:25,280 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:26,279 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:26,882 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:27,282 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:28,283 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:28,817 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:29,285 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:30,288 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:31,284 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:32,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:33,285 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:34,215 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:34,286 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:35,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:36,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:37,288 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:37,885 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:09:38,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:38,578 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:09:38,578 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:09:38,578 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:09:38,885 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:38,886 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:09:39,293 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:39,805 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:39,898 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:09:40,205 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:09:40,205 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:09:40,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:40,886 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:41,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:42,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:42,887 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:43,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:44,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:45,213 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:45,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:46,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:47,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:48,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:49,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:50,214 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:50,289 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:51,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:52,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:53,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:54,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:55,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:09:55,204 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:09:55,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:55,333 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:09:56,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:57,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:58,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:09:58,892 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:09:59,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:00,290 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:00,472 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:01,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:02,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:02,894 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:10:03,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:04,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:05,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:06,214 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:06,291 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:06,895 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:10:07,292 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:08,292 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:09,292 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:09,899 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:10:10,204 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:10:10,204 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:10:10,292 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:11,292 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:11,310 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:12,292 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:13,293 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:14,293 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:15,293 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:16,293 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:16,313 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:17,300 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:17,614 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:10:17,614 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 with policy now +2025-01-07 20:10:17,616 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:10:17,617 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:10:17,617 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:17,627 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:17,882 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/0c7ap176-media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 +2025-01-07 20:10:17,899 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:10:17,899 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 +2025-01-07 20:10:17,899 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval +2025-01-07 20:10:18,191 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/08idz8if-media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 +2025-01-07 20:10:18,311 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:18,899 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:10:19,302 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:20,308 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:21,309 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:21,381 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:22,304 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:22,900 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:10:23,310 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:24,314 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:25,210 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:10:25,210 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:10:25,309 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:26,317 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:26,902 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:10:27,312 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:27,369 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:28,311 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:29,314 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:30,325 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:31,329 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:32,326 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:33,227 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:33,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:34,327 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:35,324 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:36,345 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:36,648 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:10:36,648 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:36,649 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:36,756 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:10:36,756 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:10:36,905 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:10:36,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:10:36,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:10:36,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media +2025-01-07 20:10:36,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table +2025-01-07 20:10:36,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval +2025-01-07 20:10:36,950 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:10:37,145 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/u8grp3lf-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:10:37,170 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/lk3h76u6-events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:10:37,266 INFO wandb-upload_2:2220142 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzcs05d44 +2025-01-07 20:10:37,321 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:37,375 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:10:37,421 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:10:37,705 INFO SenderThread:2220142 [sender.py:send_request_log_artifact():1525] logged artifact run-8gt3cc6e-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM1ODYzNg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgyNDY0MQ==', 'latestArtifact': None}} +2025-01-07 20:10:37,705 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:10:37,705 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 20:10:37,705 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:10:37,705 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,706 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,706 DEBUG SenderThread:2220142 [sender.py:send():379] send: telemetry +2025-01-07 20:10:37,706 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,710 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,710 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,710 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,710 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,711 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,711 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,711 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,711 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,712 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,712 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,712 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,712 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,713 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,713 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,713 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,713 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,714 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,714 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,714 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,714 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:37,715 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:37,715 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:10:37,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:10:37,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:10:37,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json +2025-01-07 20:10:37,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval +2025-01-07 20:10:37,969 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/8j0kqxm8-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:10:38,088 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/4n1dp5b1-media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json +2025-01-07 20:10:38,093 INFO wandb-upload_2:2220142 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppc617cxg +2025-01-07 20:10:38,321 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:38,538 INFO SenderThread:2220142 [sender.py:send_request_log_artifact():1525] logged artifact run-8gt3cc6e-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM1ODY1OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgyNDY0NQ==', 'latestArtifact': None}} +2025-01-07 20:10:38,539 DEBUG SenderThread:2220142 [sender.py:send():379] send: files +2025-01-07 20:10:38,539 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json with policy now +2025-01-07 20:10:38,539 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:10:38,539 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:38,540 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:38,540 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:38,830 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/2wg74seq-media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json +2025-01-07 20:10:38,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:10:38,906 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:10:39,321 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:39,899 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:10:40,206 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:10:40,206 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:10:40,321 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:41,321 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:42,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:43,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:44,305 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:44,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:45,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:46,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:47,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:48,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:49,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:50,220 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:50,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:51,322 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:52,323 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:10:52,437 DEBUG SenderThread:2220142 [sender.py:send():379] send: exit +2025-01-07 20:10:52,438 INFO SenderThread:2220142 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:10:52,438 INFO SenderThread:2220142 [sender.py:send_exit():588] handling runtime: 342 +2025-01-07 20:10:52,438 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:52,438 INFO SenderThread:2220142 [sender.py:send_exit():594] send defer +2025-01-07 20:10:52,438 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:52,438 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:10:52,439 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:52,439 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:10:52,439 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:10:52,439 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:52,439 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:10:52,439 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:52,439 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:10:52,439 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:10:52,439 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:52,439 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:10:52,439 INFO HandlerThread:2220142 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:10:52,439 INFO HandlerThread:2220142 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:10:52,440 INFO HandlerThread:2220142 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:10:52,440 DEBUG SystemMonitor:2220142 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:10:52,440 DEBUG SystemMonitor:2220142 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:10:52,494 INFO HandlerThread:2220142 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:10:52,494 INFO HandlerThread:2220142 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:10:52,494 INFO HandlerThread:2220142 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:10:52,494 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:52,494 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:10:52,494 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:10:52,494 DEBUG SenderThread:2220142 [sender.py:send():379] send: stats +2025-01-07 20:10:52,495 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:52,495 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:10:52,495 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:52,495 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:10:52,495 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:10:52,495 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:52,495 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:10:52,911 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:10:54,912 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:10:58,656 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:10:58,656 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:58,656 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:58,656 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:10:58,656 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:10:58,657 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:10:58,658 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:10:58,658 DEBUG SenderThread:2220142 [sender.py:send():379] send: history +2025-01-07 20:10:58,658 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:10:58,658 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:58,658 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:10:58,659 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:58,659 DEBUG SenderThread:2220142 [sender.py:send():379] send: summary +2025-01-07 20:10:58,663 INFO SenderThread:2220142 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:10:58,663 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:58,663 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:10:58,663 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:10:58,663 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:58,663 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:10:58,663 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:58,663 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:10:58,667 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:10:58,807 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:10:58,807 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:10:58,807 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:10:58,807 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:10:58,807 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:10:58,914 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/config.yaml +2025-01-07 20:10:58,914 INFO Thread-12 :2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:10:59,439 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:11:02,357 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:11:02,357 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:11:02,357 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:11:02,358 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:11:02,358 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:11:02,358 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:11:02,358 INFO SenderThread:2220142 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:11:02,360 INFO SenderThread:2220142 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:11:02,362 INFO SenderThread:2220142 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:11:02,374 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:11:02,374 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:11:02,374 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:11:02,374 DEBUG SenderThread:2220142 [sender.py:send():379] send: artifact +2025-01-07 20:11:02,440 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:11:02,558 INFO SenderThread:2220142 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc1Nzg0Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:11:02,559 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:11:02,559 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:11:02,559 INFO SenderThread:2220142 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:11:02,915 INFO SenderThread:2220142 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:11:02,915 INFO SenderThread:2220142 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files +2025-01-07 20:11:02,915 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/config.yaml config.yaml +2025-01-07 20:11:02,916 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json wandb-summary.json +2025-01-07 20:11:02,916 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/requirements.txt requirements.txt +2025-01-07 20:11:02,916 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:11:02,916 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:11:02,916 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log output.log +2025-01-07 20:11:02,919 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:11:02,921 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 media/videos/eval/1_eval_1_4e6b8d6b47dcf1f2852e.mp4 +2025-01-07 20:11:02,921 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 media/videos/eval/3_eval_3_b35454858e15f45db555.mp4 +2025-01-07 20:11:02,921 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 media/videos/eval/2_eval_2_f300fd7a4cb461e89098.mp4 +2025-01-07 20:11:02,922 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 media/videos/eval/0_eval_0_94f52a23cf863a3c1137.mp4 +2025-01-07 20:11:02,922 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 media/videos/eval/4_eval_4_93d560b9b8a81ed37bb5.mp4 +2025-01-07 20:11:02,923 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json media/table/eval/total_success_rate_ci_6_130e835d6612d43dcccd.table.json +2025-01-07 20:11:02,923 INFO SenderThread:2220142 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:11:02,923 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:11:02,924 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:11:02,925 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:11:02,925 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:11:02,925 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:11:02,925 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:11:02,925 INFO SenderThread:2220142 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:11:03,108 INFO wandb-upload_0:2220142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/wandb-summary.json +2025-01-07 20:11:03,117 INFO wandb-upload_2:2220142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/config.yaml +2025-01-07 20:11:03,119 INFO wandb-upload_4:2220142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/output.log +2025-01-07 20:11:03,126 INFO wandb-upload_1:2220142 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/files/requirements.txt +2025-01-07 20:11:03,224 INFO wandb-upload_3:2220142 [upload_job.py:push():130] Uploaded file /tmp/tmplx5lrfugwandb/oc0glxge-events.out.tfevents.1736298310.leonmkim-ROG-Strix-G15CS-G15CS.2219968.0 +2025-01-07 20:11:03,424 INFO Thread-11 (_thread_body):2220142 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:11:03,424 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:11:03,424 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:11:03,424 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:11:03,425 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:11:03,425 INFO SenderThread:2220142 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:11:03,426 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:11:03,426 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:11:03,426 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:11:03,426 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:11:03,426 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:11:03,426 INFO SenderThread:2220142 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:11:03,440 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:11:03,649 INFO SenderThread:2220142 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:11:03,650 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:11:03,650 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:11:03,650 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:11:03,650 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:11:03,650 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:11:03,650 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:11:03,650 INFO SenderThread:2220142 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:11:03,650 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:11:03,650 INFO HandlerThread:2220142 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:11:03,651 DEBUG SenderThread:2220142 [sender.py:send():379] send: final +2025-01-07 20:11:03,651 DEBUG SenderThread:2220142 [sender.py:send():379] send: footer +2025-01-07 20:11:03,651 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: defer +2025-01-07 20:11:03,651 INFO SenderThread:2220142 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:11:03,651 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:11:03,651 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:11:03,652 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:11:03,652 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:11:03,652 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:11:03,652 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:11:03,652 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:11:03,654 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:03,703 DEBUG SenderThread:2220142 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:11:03,704 INFO MainThread:2220142 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:11:03,704 INFO MainThread:2220142 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:11:03,704 INFO MainThread:2220142 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:11:03,704 DEBUG HandlerThread:2220142 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:11:03,705 INFO HandlerThread:2220142 [handler.py:finish():882] shutting down handler +2025-01-07 20:11:04,652 INFO WriterThread:2220142 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/run-8gt3cc6e.wandb +2025-01-07 20:11:04,704 INFO SenderThread:2220142 [sender.py:finish():1615] shutting down sender +2025-01-07 20:11:04,704 INFO SenderThread:2220142 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:11:04,704 INFO SenderThread:2220142 [file_pusher.py:join():175] waiting for file pusher diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/logs/debug.log b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a96f32c1f152a3339eae44fc3cc183d5740d6870 --- /dev/null +++ b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Configure stats pid to 2219968 +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/settings +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:05:09,478 INFO MainThread:2219968 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:05:09,479 INFO MainThread:2219968 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/logs/debug.log +2025-01-07 20:05:09,479 INFO MainThread:2219968 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/logs/debug-internal.log +2025-01-07 20:05:09,479 INFO MainThread:2219968 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:05:09,479 INFO MainThread:2219968 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e'} +2025-01-07 20:05:09,479 INFO MainThread:2219968 [wandb_init.py:init():619] starting backend +2025-01-07 20:05:09,479 INFO MainThread:2219968 [wandb_init.py:init():623] setting up manager +2025-01-07 20:05:09,485 INFO MainThread:2219968 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:05:09,486 INFO MainThread:2219968 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:05:09,500 INFO MainThread:2219968 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:05:09,507 INFO MainThread:2219968 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:05:09,811 INFO MainThread:2219968 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:05:09,878 INFO MainThread:2219968 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:05:09,878 INFO MainThread:2219968 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:05:10,197 INFO MainThread:2219968 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:05:10,197 INFO MainThread:2219968 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:05:10,197 INFO MainThread:2219968 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:05:10,197 INFO MainThread:2219968 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:05:10,199 INFO MainThread:2219968 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:05:10,199 INFO MainThread:2219968 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/8gt3cc6e/tb, True +2025-01-07 20:05:14,001 INFO MainThread:2219968 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:11:04,705 WARNING MsgRouterThr:2219968 [router.py:message_loop():77] message_loop has been closed diff --git a/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/run-8gt3cc6e.wandb b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/run-8gt3cc6e.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5297fa2de53c8aeb5709a86a52aa8609877af690 Binary files /dev/null and b/8gt3cc6e/wandb/run-20250107_200509-8gt3cc6e/run-8gt3cc6e.wandb differ diff --git a/jv09ibrn/.hydra/hydra.yaml b/jv09ibrn/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b4d81e42e3f916f0feaf208ef8d0449c476e3fe --- /dev/null +++ b/jv09ibrn/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/jv09ibrn + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/jv09ibrn/eval_policy.log b/jv09ibrn/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..d11bcdaab9c75b93ae4ae1ed7229766096ca610f --- /dev/null +++ b/jv09ibrn/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-07 18:28:18,463][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 18:28:18,467][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-07 18:28:22,395][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/m386gyxa/episode_rosbags/episode_0_2025-01-06-20-06-21.bag b/m386gyxa/episode_rosbags/episode_0_2025-01-06-20-06-21.bag new file mode 100644 index 0000000000000000000000000000000000000000..5a3b134f8fc54e65e06e23056dd7fa79b9d004dd --- /dev/null +++ b/m386gyxa/episode_rosbags/episode_0_2025-01-06-20-06-21.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f617857ada90bf16a3f230210baad265bb1e821ad63a9677c3b82df6635f3404 +size 2605116073 diff --git a/m386gyxa/episode_rosbags/episode_1_2025-01-06-20-07-09.bag b/m386gyxa/episode_rosbags/episode_1_2025-01-06-20-07-09.bag new file mode 100644 index 0000000000000000000000000000000000000000..a270d7328589291d6b69ab6f902d9d4a9fd9c5ad --- /dev/null +++ b/m386gyxa/episode_rosbags/episode_1_2025-01-06-20-07-09.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59b3a6aa9a6e3462e01e0a16e8a60b19c812bc4547c2bc8d067a86af26d5567c +size 2614149304 diff --git a/m386gyxa/episode_rosbags/episode_3_2025-01-06-20-08-47.bag b/m386gyxa/episode_rosbags/episode_3_2025-01-06-20-08-47.bag new file mode 100644 index 0000000000000000000000000000000000000000..f5d6817c6272138168006eeb613d2f2ae8d4f871 --- /dev/null +++ b/m386gyxa/episode_rosbags/episode_3_2025-01-06-20-08-47.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d848b5f7fe1410636fa26d424f788af44c1373a3af76c35ad1f1ebe220946cab +size 2605757996 diff --git a/m386gyxa/episode_rosbags/episode_4_2025-01-06-20-09-35.bag b/m386gyxa/episode_rosbags/episode_4_2025-01-06-20-09-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..789f9714677a2c1ff9882d4e61204afe293d57de --- /dev/null +++ b/m386gyxa/episode_rosbags/episode_4_2025-01-06-20-09-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9c3ff738af0106dd7b94569ae0c127705ae4e77e4ea501fe12dbd8bb64efcbd +size 2606776917 diff --git a/up4gjipz/episode_rosbags/episode_0_2025-01-06-20-01-17.bag b/up4gjipz/episode_rosbags/episode_0_2025-01-06-20-01-17.bag new file mode 100644 index 0000000000000000000000000000000000000000..9d7a5c6bcbf910b50618e3cd669b0f5eda8a0f9c --- /dev/null +++ b/up4gjipz/episode_rosbags/episode_0_2025-01-06-20-01-17.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec47d78f6d1fad724c2d6720250818b4bf40a91df2bc4b21e1bb6565351edce2 +size 2604704010 diff --git a/vb47oelh/.hydra/config.yaml b/vb47oelh/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd5530cb9a8f25d6d55135607d04c2061b4351d1 --- /dev/null +++ b/vb47oelh/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1017_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/vb47oelh/.hydra/hydra.yaml b/vb47oelh/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bdfb0fbee361302e462e02cec3e88e7639be3c18 --- /dev/null +++ b/vb47oelh/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/vb47oelh/.hydra/overrides.yaml b/vb47oelh/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/vb47oelh/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/vb47oelh/episode_rosbags/aligned_depth_to_color_K.npy b/vb47oelh/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/vb47oelh/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/vb47oelh/episode_rosbags/cam_tf_world.npy b/vb47oelh/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/vb47oelh/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/vb47oelh/episode_rosbags/color_K.npy b/vb47oelh/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/vb47oelh/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/vb47oelh/episode_rosbags/depth_K.npy b/vb47oelh/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/vb47oelh/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/vb47oelh/eval_robot.log b/vb47oelh/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..c4c5f6cc69fdc14fbdd4d37d840291282bbd7af9 --- /dev/null +++ b/vb47oelh/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 18:06:13,246][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 18:06:13,249][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/vb47oelh/eval_video/0_eval.mp4 b/vb47oelh/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bbe8bfa47422959cc09251ac5b485d02ce0dd964 --- /dev/null +++ b/vb47oelh/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a21f553da3e495971337afb6161b7df0b7b49ff3877bdbbaebf22b37f68ebbdc +size 1439309 diff --git a/vb47oelh/eval_video/1_eval.mp4 b/vb47oelh/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..869a03a6177db87dc25ce5d74a0a13c7aba5303b --- /dev/null +++ b/vb47oelh/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbe5d527784bc078331e483e656bc450b6f817a935395c86e3dafc3ad965c571 +size 1458271 diff --git a/vb47oelh/eval_video/2_eval.mp4 b/vb47oelh/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c9af224777331ffcb03e963d355aba05a89ea373 --- /dev/null +++ b/vb47oelh/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71bebcde765f8093e4dbd59ab69eccffbf75031cb294f2664ecb027986fd93b5 +size 1489977 diff --git a/vb47oelh/eval_video/3_eval.mp4 b/vb47oelh/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ea9770d79184e5b339f1559701844f1e9a490a7d --- /dev/null +++ b/vb47oelh/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9667046e025c1a04f58455d2c680ccbca3f09203b9ed7ab369196b94ad1485db +size 1485281 diff --git a/vb47oelh/eval_video/4_eval.mp4 b/vb47oelh/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c209b756e4e333d19f6204c84d708c4f59c8645 --- /dev/null +++ b/vb47oelh/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:537861e0f91853c73080f9a092a55c7700e3f0eb015781904701d74188a5fb2f +size 1569964 diff --git a/vb47oelh/tb/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 b/vb47oelh/tb/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 new file mode 100644 index 0000000000000000000000000000000000000000..1b4fffd09034df551367ea8fb133f1d1afa3ecb1 --- /dev/null +++ b/vb47oelh/tb/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4476c92424c8118173827387fabcbb452bd46167e1f42dc1210a03b4a22d50d +size 6539 diff --git a/vb47oelh/wandb/debug-internal.log b/vb47oelh/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..088157eccb809a8af510fd883a957032fa7c818b --- /dev/null +++ b/vb47oelh/wandb/debug-internal.log @@ -0,0 +1,861 @@ +2025-01-06 18:06:19,073 INFO StreamThr :2098647 [internal.py:wandb_internal():85] W&B internal server running at pid: 2098647, started at: 2025-01-06 18:06:19.072755 +2025-01-06 18:06:19,074 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status +2025-01-06 18:06:19,085 INFO WriterThread:2098647 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/run-vb47oelh.wandb +2025-01-06 18:06:19,085 DEBUG SenderThread:2098647 [sender.py:send():379] send: header +2025-01-06 18:06:19,094 DEBUG SenderThread:2098647 [sender.py:send():379] send: run +2025-01-06 18:06:19,421 INFO SenderThread:2098647 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files +2025-01-06 18:06:19,421 INFO SenderThread:2098647 [sender.py:_start_run_threads():1188] run started: vb47oelh with start time 1736204779.072731 +2025-01-06 18:06:19,438 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 18:06:19,438 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: check_version +2025-01-06 18:06:19,509 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 18:06:19,514 DEBUG HandlerThread:2098647 [system_info.py:__init__():26] System info init +2025-01-06 18:06:19,514 DEBUG HandlerThread:2098647 [system_info.py:__init__():41] System info init done +2025-01-06 18:06:19,514 INFO HandlerThread:2098647 [system_monitor.py:start():194] Starting system monitor +2025-01-06 18:06:19,514 INFO SystemMonitor:2098647 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 18:06:19,514 INFO HandlerThread:2098647 [system_monitor.py:probe():214] Collecting system info +2025-01-06 18:06:19,515 INFO SystemMonitor:2098647 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 18:06:19,516 INFO SystemMonitor:2098647 [interfaces.py:start():188] Started disk monitoring +2025-01-06 18:06:19,517 INFO SystemMonitor:2098647 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 18:06:19,517 INFO SystemMonitor:2098647 [interfaces.py:start():188] Started memory monitoring +2025-01-06 18:06:19,518 INFO SystemMonitor:2098647 [interfaces.py:start():188] Started network monitoring +2025-01-06 18:06:19,526 DEBUG HandlerThread:2098647 [system_info.py:probe():152] Probing system +2025-01-06 18:06:19,528 DEBUG HandlerThread:2098647 [system_info.py:_probe_git():137] Probing git +2025-01-06 18:06:19,533 DEBUG HandlerThread:2098647 [system_info.py:_probe_git():145] Probing git done +2025-01-06 18:06:19,533 DEBUG HandlerThread:2098647 [system_info.py:probe():200] Probing system done +2025-01-06 18:06:19,533 DEBUG HandlerThread:2098647 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T23:06:19.526838', 'startedAt': '2025-01-06T23:06:19.061983', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.11675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 4500.058, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 401.7840385437012}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 18:06:19,533 INFO HandlerThread:2098647 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 18:06:19,533 INFO HandlerThread:2098647 [system_monitor.py:probe():227] Publishing system info +2025-01-06 18:06:19,533 DEBUG HandlerThread:2098647 [system_info.py:_save_code():44] Saving code +2025-01-06 18:06:19,534 DEBUG HandlerThread:2098647 [system_info.py:_save_code():67] Saving code done +2025-01-06 18:06:19,534 DEBUG HandlerThread:2098647 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 18:06:19,596 DEBUG HandlerThread:2098647 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 18:06:19,596 INFO HandlerThread:2098647 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 18:06:19,599 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:06:19,599 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 18:06:19,600 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 18:06:19,600 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 18:06:19,838 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 18:06:19,838 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: python_packages +2025-01-06 18:06:19,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:06:19,839 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:06:19,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:19,842 INFO HandlerThread:2098647 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/tb" + save: true +} +control { + relay_id: "4213779cbc0c4ea28496f8328555bbcc" +} +_info { + stream_id: "vb47oelh" +} + +2025-01-06 18:06:19,842 DEBUG HandlerThread:2098647 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 18:06:19,849 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/28oyl0y4-wandb-metadata.json +2025-01-06 18:06:19,876 INFO wandb-upload_1:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/3qzd1pu5-code/FISH/eval_robot.py +2025-01-06 18:06:19,887 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/pgurgypv-diff.patch +2025-01-06 18:06:19,996 DEBUG SenderThread:2098647 [sender.py:send():379] send: telemetry +2025-01-06 18:06:19,996 DEBUG SenderThread:2098647 [sender.py:send():379] send: tbrecord +2025-01-06 18:06:19,997 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:06:19,997 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 with policy live +2025-01-06 18:06:20,182 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/qm58b42e-events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:06:20,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/requirements.txt +2025-01-06 18:06:20,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/diff.patch +2025-01-06 18:06:20,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/code/FISH/eval_robot.py +2025-01-06 18:06:20,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:06:20,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-metadata.json +2025-01-06 18:06:20,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:06:20,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/code/FISH +2025-01-06 18:06:20,424 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/code +2025-01-06 18:06:20,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:21,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:22,422 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:06:22,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:23,669 DEBUG SenderThread:2098647 [sender.py:send():379] send: config +2025-01-06 18:06:23,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:24,677 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:06:24,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:25,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:26,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:06:26,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:27,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:28,424 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:06:28,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:29,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:06:29,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:30,424 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:06:30,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:31,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:32,425 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:06:32,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:33,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:34,838 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:06:34,839 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:06:34,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:34,995 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:06:35,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:36,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:37,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:38,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:39,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:40,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:06:40,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:41,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:42,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:43,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:44,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:45,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:46,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:06:46,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:47,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:48,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:49,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:06:49,839 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:06:49,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:50,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:51,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:51,929 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:06:52,430 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/config.yaml +2025-01-06 18:06:52,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:53,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:54,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:55,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:56,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:57,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:06:57,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:58,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:59,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:00,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:01,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:02,433 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:07:02,846 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:03,471 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:03,846 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:04,434 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:07:04,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:07:04,839 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:07:04,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:05,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:06,434 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:07:06,846 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:07,846 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:08,846 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:08,928 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:09,847 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:10,847 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:11,847 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:12,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:13,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:14,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:14,849 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:15,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:16,849 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:17,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:18,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:19,518 DEBUG SystemMonitor:2098647 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 18:07:19,519 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:07:19,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:07:19,841 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:07:19,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:19,918 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:20,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:21,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:22,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:23,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:24,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:25,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:25,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:26,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:27,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:28,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:29,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:30,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:30,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:31,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:32,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:33,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:34,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:07:34,841 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:07:34,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:35,854 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:36,151 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:36,853 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:37,854 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:38,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:39,469 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:07:39,469 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 with policy now +2025-01-06 18:07:39,471 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:07:39,472 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:07:39,472 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:07:39,473 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:07:39,719 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/k9t2peu7-media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 +2025-01-06 18:07:39,854 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:40,443 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:07:40,443 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 +2025-01-06 18:07:40,443 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:07:40,444 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval +2025-01-06 18:07:40,444 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos +2025-01-06 18:07:40,444 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media +2025-01-06 18:07:40,718 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/khbhrbwo-media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 +2025-01-06 18:07:40,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:41,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:41,857 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:42,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:43,854 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:44,445 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:07:44,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:45,854 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:46,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:47,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:47,858 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:48,446 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:07:48,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:49,519 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:07:49,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:07:49,841 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:07:49,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:50,864 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:51,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:52,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:52,995 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:53,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:54,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:55,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:56,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:57,449 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:07:57,701 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/fboc6tt8-events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:07:57,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:58,449 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:07:58,677 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:58,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:59,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:00,450 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:00,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:01,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:02,451 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:02,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:03,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:04,327 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:04,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:08:04,842 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:08:04,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:05,857 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:06,857 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:07,857 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:08,858 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:09,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:09,857 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:10,859 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:11,858 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:12,858 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:13,864 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:14,454 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:14,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:14,859 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:15,860 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:16,860 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:17,860 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:18,861 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:19,519 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:08:19,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:08:19,842 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:08:19,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:19,983 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:20,861 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:21,862 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:22,457 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:22,861 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:23,862 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:24,862 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:25,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:25,865 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:26,863 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:27,863 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:28,863 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:29,863 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:30,459 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:30,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:30,863 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:31,864 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:32,863 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:33,863 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:34,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:08:34,843 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:08:34,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:35,869 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:36,349 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:36,461 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:36,618 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:08:36,618 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 with policy now +2025-01-06 18:08:36,623 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:08:36,624 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:08:36,624 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:08:36,729 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:08:36,845 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/di80x26n-media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 +2025-01-06 18:08:36,873 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:37,461 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:08:37,462 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 +2025-01-06 18:08:37,462 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval +2025-01-06 18:08:37,729 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/5v4ah568-media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 +2025-01-06 18:08:37,876 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:38,462 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:38,870 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:39,875 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:40,871 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:41,871 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:42,313 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:42,463 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:42,875 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:43,874 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:44,464 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:44,873 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:45,879 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:46,880 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:47,848 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:47,880 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:48,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:49,466 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:08:49,520 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:08:49,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:08:49,843 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:08:49,878 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:50,309 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:08:50,309 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:08:50,309 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:08:50,466 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:50,467 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:08:50,879 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:51,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:52,467 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:52,882 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:53,166 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:53,883 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:54,468 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:54,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:55,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:56,469 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:56,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:57,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:58,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:58,934 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:59,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:00,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:01,882 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:02,883 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:03,883 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:04,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:09:04,842 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:09:04,889 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:04,980 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:05,883 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:06,883 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:07,884 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:08,883 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:09,884 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:10,848 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:10,884 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:11,884 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:12,884 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:13,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:14,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:15,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:16,848 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:16,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:17,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:18,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:19,520 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:09:19,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:09:19,842 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:09:19,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:20,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:21,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:21,973 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:22,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:23,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:24,886 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:25,886 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:26,886 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:27,848 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:27,886 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:28,886 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:29,895 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:30,480 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:09:30,626 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:09:30,626 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 with policy now +2025-01-06 18:09:30,637 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:09:30,638 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:09:30,638 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:09:30,660 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:09:30,856 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/ircl3c9e-media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 +2025-01-06 18:09:30,900 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:31,481 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:09:31,481 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 +2025-01-06 18:09:31,481 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval +2025-01-06 18:09:31,746 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/8t37ch96-media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 +2025-01-06 18:09:31,896 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:32,481 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:09:32,902 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:33,675 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:33,897 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:34,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:09:34,851 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:09:34,897 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:35,910 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:36,482 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:09:36,902 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:37,901 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:38,483 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:09:38,899 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:39,249 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:39,908 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:40,909 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:41,909 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:42,906 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:43,151 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:09:43,151 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:09:43,152 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:09:43,486 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:09:43,486 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:09:43,736 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/c73vqglr-events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:09:43,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:44,487 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:09:44,705 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:44,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:45,916 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:46,908 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:47,909 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:48,489 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:09:48,908 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:49,521 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:09:49,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:09:49,844 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:09:49,909 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:49,940 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:50,490 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:09:50,909 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:51,909 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:52,910 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:53,910 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:54,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:55,853 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:55,911 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:56,911 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:57,911 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:58,911 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:59,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:00,853 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:00,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:01,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:02,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:03,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:04,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:10:04,843 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:10:04,913 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:05,913 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:05,994 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:06,913 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:07,913 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:08,913 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:09,913 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:10,914 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:11,854 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:11,914 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:12,914 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:13,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:14,914 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:15,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:16,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:17,854 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:17,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:18,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:19,521 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:10:19,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:10:19,843 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:10:19,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:20,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:21,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:22,500 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:22,921 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:23,406 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:23,695 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:10:23,695 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 with policy now +2025-01-06 18:10:23,697 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:10:23,698 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:10:23,698 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:10:23,714 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:10:23,923 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:23,931 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/lcary7gx-media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 +2025-01-06 18:10:24,501 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:24,501 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:10:24,501 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 +2025-01-06 18:10:24,502 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval +2025-01-06 18:10:24,764 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/7ysestke-media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 +2025-01-06 18:10:24,924 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:25,935 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:26,923 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:27,926 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:28,437 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:28,502 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:28,923 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:29,928 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:30,926 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:31,927 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:32,504 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:32,926 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:33,933 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:34,233 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:34,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:10:34,851 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:10:34,936 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:35,929 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:36,930 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:37,945 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:38,938 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:39,861 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:39,936 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:40,933 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:41,223 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:10:41,223 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:10:41,224 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:10:41,507 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:10:41,507 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:10:41,945 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:42,507 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:42,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:43,945 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:44,508 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:44,945 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:45,858 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:45,954 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:46,509 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:46,946 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:47,946 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:48,509 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:48,946 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:49,522 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:10:49,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:10:49,845 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:10:49,946 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:50,946 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:50,996 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:51,946 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:52,946 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:53,947 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:54,947 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:55,947 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:56,857 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:56,947 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:57,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:58,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:59,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:00,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:01,857 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:01,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:02,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:03,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:04,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:11:04,845 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:11:04,949 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:05,949 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:06,927 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:06,949 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:07,949 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:08,949 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:09,949 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:10,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:11,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:12,859 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:12,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:13,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:14,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:15,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:16,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:17,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:18,859 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:18,951 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:19,523 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:11:19,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:11:19,845 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:11:19,951 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:20,951 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:21,951 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:22,957 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:23,212 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:11:23,212 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 with policy now +2025-01-06 18:11:23,216 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:11:23,216 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:11:23,217 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:23,218 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:23,439 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/3oo7pnjy-media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 +2025-01-06 18:11:23,523 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:11:23,523 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 +2025-01-06 18:11:23,523 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval +2025-01-06 18:11:23,787 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/yc2x18el-media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 +2025-01-06 18:11:23,967 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:24,231 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:24,523 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:24,955 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:25,979 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:26,969 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:27,963 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:28,524 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:28,966 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:29,688 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:29,962 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:30,525 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:30,959 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:31,967 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:32,526 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:11:32,766 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/j460tiwh-events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:11:32,964 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:33,287 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:11:33,287 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:33,287 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:33,351 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 18:11:33,351 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 18:11:33,526 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:11:33,526 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:11:33,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media +2025-01-06 18:11:33,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval +2025-01-06 18:11:33,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table +2025-01-06 18:11:33,540 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:11:33,697 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/qfa78be5-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:11:33,845 INFO wandb-upload_2:2098647 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6qmyzqmn +2025-01-06 18:11:33,923 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 18:11:33,931 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:11:33,956 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:34,227 INFO SenderThread:2098647 [sender.py:send_request_log_artifact():1525] logged artifact run-vb47oelh-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTk5MDU4NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxODI0OA==', 'latestArtifact': None}} +2025-01-06 18:11:34,227 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:11:34,227 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 18:11:34,227 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:11:34,227 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,228 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,228 DEBUG SenderThread:2098647 [sender.py:send():379] send: telemetry +2025-01-06 18:11:34,228 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,232 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,232 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,232 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,232 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,233 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,233 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,233 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,233 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,234 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,234 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,234 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,235 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,235 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,235 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,235 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,236 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,236 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,236 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,236 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,236 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,237 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,237 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 18:11:34,467 INFO wandb-upload_1:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/o8lfi9aq-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:11:34,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:34,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:11:34,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:11:34,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 18:11:34,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval +2025-01-06 18:11:34,649 INFO wandb-upload_2:2098647 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqycxkxgi +2025-01-06 18:11:34,789 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/ye57dor5-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 18:11:34,846 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:11:34,956 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:35,024 INFO SenderThread:2098647 [sender.py:send_request_log_artifact():1525] logged artifact run-vb47oelh-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTk5MDU5OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxODI0OQ==', 'latestArtifact': None}} +2025-01-06 18:11:35,024 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:11:35,025 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:35,025 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json with policy now +2025-01-06 18:11:35,025 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:11:35,025 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:35,026 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:35,026 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:11:35,271 INFO wandb-upload_1:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/f9qg76k0-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 18:11:35,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:11:35,956 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:36,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:36,956 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:37,957 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:38,957 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:39,957 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:40,873 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:40,958 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:41,957 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:42,958 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:43,958 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:44,958 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:45,873 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:45,960 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:46,960 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:47,958 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:48,959 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:49,523 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:11:49,846 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:11:49,846 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:11:49,958 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:50,914 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:51,184 DEBUG SenderThread:2098647 [sender.py:send():379] send: exit +2025-01-06 18:11:51,184 INFO SenderThread:2098647 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 18:11:51,184 INFO SenderThread:2098647 [sender.py:send_exit():588] handling runtime: 331 +2025-01-06 18:11:51,184 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:51,184 INFO SenderThread:2098647 [sender.py:send_exit():594] send defer +2025-01-06 18:11:51,184 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:51,185 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 18:11:51,185 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:51,185 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 18:11:51,185 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 1 +2025-01-06 18:11:51,185 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:51,185 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 18:11:51,185 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:51,185 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 18:11:51,185 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 2 +2025-01-06 18:11:51,185 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:51,186 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 18:11:51,186 INFO HandlerThread:2098647 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 18:11:51,186 DEBUG SystemMonitor:2098647 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 18:11:51,186 DEBUG SystemMonitor:2098647 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 18:11:51,186 INFO HandlerThread:2098647 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 18:11:51,187 INFO HandlerThread:2098647 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 18:11:51,221 INFO HandlerThread:2098647 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 18:11:51,221 INFO HandlerThread:2098647 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 18:11:51,221 INFO HandlerThread:2098647 [interfaces.py:finish():200] Joined network monitor +2025-01-06 18:11:51,222 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:51,222 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 18:11:51,222 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 3 +2025-01-06 18:11:51,222 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:11:51,222 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:51,222 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 18:11:51,222 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:51,222 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 18:11:51,222 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 4 +2025-01-06 18:11:51,223 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:51,223 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 18:11:51,534 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/config.yaml +2025-01-06 18:11:51,534 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:11:52,535 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:57,292 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:11:57,292 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:57,292 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:57,292 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 18:11:57,292 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 5 +2025-01-06 18:11:57,292 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 18:11:57,292 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:11:57,293 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:57,294 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 18:11:57,293 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:11:57,294 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:57,294 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:57,295 DEBUG SenderThread:2098647 [sender.py:send():379] send: summary +2025-01-06 18:11:57,298 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:57,299 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:57,299 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 18:11:57,299 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 6 +2025-01-06 18:11:57,299 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:57,299 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 18:11:57,299 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:57,299 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 18:11:57,299 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 7 +2025-01-06 18:11:57,299 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:57,299 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:57,299 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 18:11:57,300 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:57,300 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 18:11:57,537 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:11:57,916 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:11:58,349 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 8 +2025-01-06 18:11:58,349 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:11:58,349 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:58,349 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 18:11:58,350 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:58,350 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 18:11:58,350 INFO SenderThread:2098647 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 18:11:58,353 INFO SenderThread:2098647 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 18:11:58,355 INFO SenderThread:2098647 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 18:11:58,370 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 9 +2025-01-06 18:11:58,370 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:58,370 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 18:11:58,370 DEBUG SenderThread:2098647 [sender.py:send():379] send: artifact +2025-01-06 18:11:58,537 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:58,810 INFO wandb-upload_2:2098647 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1mspmxl3 +2025-01-06 18:11:58,811 INFO wandb-upload_1:2098647 [upload_job.py:push():85] Skipped uploading /tmp/tmprros7r4x/wandb-job.json +2025-01-06 18:11:58,916 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:11:58,939 INFO wandb-upload_0:2098647 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpntr5kgdh +2025-01-06 18:11:59,520 INFO SenderThread:2098647 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk3NDE3OA==', 'versionIndex': 426}}} +2025-01-06 18:11:59,520 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:59,520 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 18:11:59,520 INFO SenderThread:2098647 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 18:11:59,538 INFO SenderThread:2098647 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files +2025-01-06 18:11:59,538 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/config.yaml config.yaml +2025-01-06 18:11:59,538 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json wandb-summary.json +2025-01-06 18:11:59,539 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:11:59,539 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/requirements.txt requirements.txt +2025-01-06 18:11:59,539 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-metadata.json wandb-metadata.json +2025-01-06 18:11:59,539 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log output.log +2025-01-06 18:11:59,542 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/diff.patch diff.patch +2025-01-06 18:11:59,543 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 18:11:59,543 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 +2025-01-06 18:11:59,543 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 +2025-01-06 18:11:59,543 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 +2025-01-06 18:11:59,543 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 +2025-01-06 18:11:59,543 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 +2025-01-06 18:11:59,546 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 18:11:59,546 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:11:59,546 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 10 +2025-01-06 18:11:59,546 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:11:59,548 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:59,548 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 18:11:59,548 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:59,548 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 18:11:59,548 INFO SenderThread:2098647 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 18:11:59,727 INFO wandb-upload_1:2098647 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:11:59,752 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/config.yaml +2025-01-06 18:11:59,765 INFO wandb-upload_3:2098647 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/requirements.txt +2025-01-06 18:11:59,780 INFO wandb-upload_4:2098647 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:59,785 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/mmn2rth2-events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:11:59,917 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:11:59,917 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:11:59,985 INFO Thread-11 (_thread_body):2098647 [sender.py:transition_state():614] send defer: 11 +2025-01-06 18:11:59,985 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:59,985 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 18:11:59,985 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:59,985 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 18:11:59,985 INFO SenderThread:2098647 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 18:11:59,987 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 12 +2025-01-06 18:11:59,987 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:59,987 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 18:11:59,987 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:59,987 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 18:11:59,987 INFO SenderThread:2098647 [file_stream.py:finish():601] file stream finish called +2025-01-06 18:12:00,178 INFO SenderThread:2098647 [file_stream.py:finish():605] file stream finish is done +2025-01-06 18:12:00,178 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 13 +2025-01-06 18:12:00,178 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:12:00,178 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 18:12:00,178 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:12:00,178 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 18:12:00,179 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 14 +2025-01-06 18:12:00,179 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:12:00,179 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 18:12:00,179 DEBUG SenderThread:2098647 [sender.py:send():379] send: final +2025-01-06 18:12:00,179 DEBUG SenderThread:2098647 [sender.py:send():379] send: footer +2025-01-06 18:12:00,179 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:12:00,179 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 18:12:00,180 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:12:00,180 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:12:00,180 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 18:12:00,180 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: server_info +2025-01-06 18:12:00,183 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:12:00,183 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 18:12:00,184 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 18:12:00,184 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:00,232 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:12:00,232 INFO MainThread:2098647 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 18:12:00,232 INFO MainThread:2098647 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 18:12:00,233 INFO MainThread:2098647 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 18:12:00,233 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 18:12:00,233 INFO HandlerThread:2098647 [handler.py:finish():882] shutting down handler +2025-01-06 18:12:01,184 INFO WriterThread:2098647 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/run-vb47oelh.wandb +2025-01-06 18:12:01,232 INFO SenderThread:2098647 [sender.py:finish():1615] shutting down sender +2025-01-06 18:12:01,232 INFO SenderThread:2098647 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 18:12:01,232 INFO SenderThread:2098647 [file_pusher.py:join():175] waiting for file pusher diff --git a/vb47oelh/wandb/debug.log b/vb47oelh/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..96fe7c765396d8b14089af599aef60339ccfb510 --- /dev/null +++ b/vb47oelh/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Configure stats pid to 2098452 +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/settings +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 18:06:19,066 INFO MainThread:2098452 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/logs/debug.log +2025-01-06 18:06:19,066 INFO MainThread:2098452 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/logs/debug-internal.log +2025-01-06 18:06:19,066 INFO MainThread:2098452 [wandb_init.py:init():569] calling init triggers +2025-01-06 18:06:19,067 INFO MainThread:2098452 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh'} +2025-01-06 18:06:19,067 INFO MainThread:2098452 [wandb_init.py:init():619] starting backend +2025-01-06 18:06:19,067 INFO MainThread:2098452 [wandb_init.py:init():623] setting up manager +2025-01-06 18:06:19,071 INFO MainThread:2098452 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 18:06:19,072 INFO MainThread:2098452 [wandb_init.py:init():631] backend started and connected +2025-01-06 18:06:19,084 INFO MainThread:2098452 [wandb_init.py:init():720] updated telemetry +2025-01-06 18:06:19,092 INFO MainThread:2098452 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 18:06:19,437 INFO MainThread:2098452 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 18:06:19,504 INFO MainThread:2098452 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 18:06:19,504 INFO MainThread:2098452 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 18:06:19,839 INFO MainThread:2098452 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 18:06:19,839 INFO MainThread:2098452 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 18:06:19,840 INFO MainThread:2098452 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 18:06:19,840 INFO MainThread:2098452 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 18:06:19,841 INFO MainThread:2098452 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 18:06:19,841 INFO MainThread:2098452 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/tb, True +2025-01-06 18:06:23,669 INFO MainThread:2098452 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 18:12:01,233 WARNING MsgRouterThr:2098452 [router.py:message_loop():77] message_loop has been closed diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/code/FISH/eval_robot.py b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/config.yaml b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6877ed66cc1603c0ffcba1a86a856fa46be3a87f --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1017_0' +wandb_notes: + desc: null + value: 1017_0_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736204779 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/diff.patch b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..4c1a5f92eb09cddbc2402d3534362390d486902d --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..54e6cd0 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json new file mode 100644 index 0000000000000000000000000000000000000000..ebd62798ae2246ea155c51d2c7f88d715ecd78c3 --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[5, 5, 1.0, 0.43520843983992363, 1, 0.60740816591022, 1, 0.6842965185273651, 1, 0.6614799052153926, 1]]} \ No newline at end of file diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bbe8bfa47422959cc09251ac5b485d02ce0dd964 --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a21f553da3e495971337afb6161b7df0b7b49ff3877bdbbaebf22b37f68ebbdc +size 1439309 diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..869a03a6177db87dc25ce5d74a0a13c7aba5303b --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbe5d527784bc078331e483e656bc450b6f817a935395c86e3dafc3ad965c571 +size 1458271 diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c9af224777331ffcb03e963d355aba05a89ea373 --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71bebcde765f8093e4dbd59ab69eccffbf75031cb294f2664ecb027986fd93b5 +size 1489977 diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ea9770d79184e5b339f1559701844f1e9a490a7d --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9667046e025c1a04f58455d2c680ccbca3f09203b9ed7ab369196b94ad1485db +size 1485281 diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c209b756e4e333d19f6204c84d708c4f59c8645 --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:537861e0f91853c73080f9a092a55c7700e3f0eb015781904701d74188a5fb2f +size 1569964 diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..abcf2f6eaa20c3053cc0165ed43be8c1410506bd --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log @@ -0,0 +1,267 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736204783.669779]: resetting environment +[INFO] [1736204783.674333]: cleared current plan +[INFO] [1736204783.675395]: moving to home +[INFO] [1736204784.677305]: reached home +[INFO] [1736204784.678743]: reset action history +[INFO] [1736204787.389502]: environment reset +Starting episode 0 +[INFO] [1736204787.394081]: resetting environment +[INFO] [1736204787.398398]: cleared current plan +[INFO] [1736204787.399599]: moving to home +[INFO] [1736204788.401393]: reached home +[INFO] [1736204788.402748]: reset action history +[INFO] [1736204791.117498]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736204819.748976]: resetting environment +[INFO] [1736204819.754320]: cleared current plan +[INFO] [1736204819.755478]: moving to home +[INFO] [1736204820.758788]: reached home +[INFO] [1736204820.760040]: reset action history +[INFO] [1736204823.470739]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/episode_rosbags/episode_0_2025-01-06-18-07-03.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2098452.log +started roslaunch server http://158.130.50.37:38061/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2098781] +process[print_text-2]: started with pid [2098782] +process[rosbag_record-3]: started with pid [2098783] +started bagging! +[INFO] [1736204858.553495]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736204859.268905]: Stored episode 1. +[INFO] [1736204859.269192]: Saving video... +[INFO] [1736204859.458486]: Video saved! +Starting episode 1 +[INFO] [1736204859.471524]: resetting environment +[INFO] [1736204859.479495]: cleared current plan +[INFO] [1736204859.480691]: moving to home +[INFO] [1736204863.083081]: reached home +[INFO] [1736204863.083735]: reset action history +[INFO] [1736204865.794774]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736204876.622727]: resetting environment +[INFO] [1736204876.629419]: cleared current plan +[INFO] [1736204876.630284]: moving to home +[INFO] [1736204877.631800]: reached home +[INFO] [1736204877.632247]: reset action history +[INFO] [1736204880.339813]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/episode_rosbags/episode_1_2025-01-06-18-08-00.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2098452.log +started roslaunch server http://158.130.50.37:42859/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2098929] +process[print_text-5]: started with pid [2098930] +process[rosbag_record-6]: started with pid [2098931] +started bagging! +For topic gripper_width: timestamp difference is -169271564 for nearest: 1736204892040185118 - target: 1736204892209456682 at idx 148 +For topic gripper_width: timestamp difference is -228114419 for nearest: 1736204899940105386 - target: 1736204900168219805 at idx 246 +For topic gripper_width: timestamp difference is 105655243 for nearest: 1736204900806770613 - target: 1736204900701115370 at idx 249 +For topic gripper_width: timestamp difference is 111962288 for nearest: 1736204908506767481 - target: 1736204908394805193 at idx 249 +[INFO] [1736204915.485466]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736204916.348350]: Stored episode 2. +[INFO] [1736204916.351451]: Saving video... +[INFO] [1736204916.574150]: Video saved! +Starting episode 2 +[INFO] [1736204916.628250]: resetting environment +[INFO] [1736204916.660034]: cleared current plan +[INFO] [1736204916.662395]: moving to home +[INFO] [1736204920.269314]: reached home +[INFO] [1736204920.270249]: reset action history +[INFO] [1736204923.046639]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736204929.348021]: resetting environment +[INFO] [1736204929.367508]: cleared current plan +[INFO] [1736204929.369184]: moving to home +[INFO] [1736204930.372673]: reached home +[INFO] [1736204930.373172]: reset action history +[INFO] [1736204933.164803]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/episode_rosbags/episode_2_2025-01-06-18-08-53.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2098452.log +started roslaunch server http://158.130.50.37:46171/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2099071] +process[print_text-8]: started with pid [2099072] +process[rosbag_record-9]: started with pid [2099073] +started bagging! +[INFO] [1736204969.537057]: Storing episode... +[print_text-8] killing on exit +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[INFO] [1736204970.342870]: Stored episode 3. +[INFO] [1736204970.345592]: Saving video... +[INFO] [1736204970.553772]: Video saved! +Starting episode 3 +[INFO] [1736204970.640782]: resetting environment +[INFO] [1736204970.672425]: cleared current plan +[INFO] [1736204970.674041]: moving to home +[INFO] [1736204974.281151]: reached home +[INFO] [1736204974.283435]: reset action history +[INFO] [1736204977.207051]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736204982.636851]: resetting environment +[INFO] [1736204982.659096]: cleared current plan +[INFO] [1736204982.659688]: moving to home +[INFO] [1736204983.662296]: reached home +[INFO] [1736204983.666734]: reset action history +[INFO] [1736204986.453378]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/episode_rosbags/episode_3_2025-01-06-18-09-46.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2098452.log +started roslaunch server http://158.130.50.37:42059/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2099212] +process[print_text-11]: started with pid [2099213] +process[rosbag_record-12]: started with pid [2099214] +started bagging! +For topic gripper_width: timestamp difference is -106990149 for nearest: 1736205020040111730 - target: 1736205020147101879 at idx 249 +[INFO] [1736205022.597711]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736205023.404139]: Stored episode 4. +[INFO] [1736205023.413312]: Saving video... +[INFO] [1736205023.647539]: Video saved! +Starting episode 4 +[INFO] [1736205023.705202]: resetting environment +[INFO] [1736205023.763257]: cleared current plan +[INFO] [1736205023.772619]: moving to home +[INFO] [1736205027.387379]: reached home +[INFO] [1736205027.390446]: reset action history +[INFO] [1736205030.188629]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736205040.874644]: resetting environment +[INFO] [1736205040.898950]: cleared current plan +[INFO] [1736205040.899201]: moving to home +[INFO] [1736205041.901758]: reached home +[INFO] [1736205041.902835]: reset action history +[INFO] [1736205044.681257]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/episode_rosbags/episode_4_2025-01-06-18-10-44.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2098452.log +started roslaunch server http://158.130.50.37:46549/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2099355] +process[print_text-14]: started with pid [2099356] +process[rosbag_record-15]: started with pid [2099357] +started bagging! +[INFO] [1736205082.102055]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1736205082.944357]: Stored episode 5. +[INFO] [1736205082.949918]: Saving video... +[INFO] [1736205083.163378]: Video saved! +[INFO] [1736205083.216009]: resetting environment +[INFO] [1736205083.230672]: cleared current plan +[INFO] [1736205083.230908]: moving to home +[INFO] [1736205086.836548]: reached home +[INFO] [1736205086.844633]: reset action history +[INFO] [1736205089.681957]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/requirements.txt b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-metadata.json b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c099993d53bbe9da4b47113b750025e09d931f49 --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-06T23:06:19.526838", + "startedAt": "2025-01-06T23:06:19.061983", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3561.11675, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.996, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.058, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.002, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 401.7840385437012 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d6bf6dbd4f62ecf865836da32f9aec3965b415c3 --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "a21f553da3e495971337afb6161b7df0b7b49ff3877bdbbaebf22b37f68ebbdc", "size": 1439309, "path": "media/videos/eval/0_eval_0_a21f553da3e495971337.mp4"}, "global_step": 665, "_timestamp": 1736205093.8074894, "_runtime": 314.7347583770752, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "bbe5d527784bc078331e483e656bc450b6f817a935395c86e3dafc3ad965c571", "size": 1458271, "path": "media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4"}, "eval/num_success": 5.0, "episode": 5.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "71bebcde765f8093e4dbd59ab69eccffbf75031cb294f2664ecb027986fd93b5", "size": 1489977, "path": "media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "9667046e025c1a04f58455d2c680ccbca3f09203b9ed7ab369196b94ad1485db", "size": 1485281, "path": "media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "537861e0f91853c73080f9a092a55c7700e3f0eb015781904701d74188a5fb2f", "size": 1569964, "path": "media/videos/eval/4_eval_4_537861e0f91853c73080.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://l07ff6fqeukb2mmc8k4ai6mq9cwne5fa408y6kotrgxcd9p9piu9puv44gnvxlbwxdpkkitk608y6rwv8sy0u719n52s4exxiv5pqcoejor0cve1r6tybx7okkx4l0xg/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://rxbzil7ye8m5a3u3jcn389cmgd67vviawpjlsymlzxigxqpggptotxl97da8gchkcmcy99sb3r867o1vsjgyegi4ec52d1bqq44r5u5xunvdytv9mlfseirvlu57k088:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 5, "total_num_episodes": 5, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.43520843983992363, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.60740816591022, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.6842965185273651, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 0.6614799052153926, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "e32f206babc2825ce21715b938b57354f64231e089822aca554fad29505cf476", "size": 434, "artifact_path": "wandb-client-artifact://jb7zm72srrlnkwddb0dar2we2codh9zk7u0e6l94286cklmv42fbe6xykkmkj54ooe7xi1173trlvq0k8we6bydcyo1s38iiowunipz8qqyoyj8l4ma2cx2elusqrroq/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://pq8w63e5k93o0hi30mrib80pa2jc7o0he7cq90yts00yheep7vki1g06lxv11b946kt3wcs25bvvcx8eqtwfo6p0rg9bqw612jr36z7shm3zod1wo0s4bn7r1ssuez8k:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 331}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.43520843982696533, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.6074081659317017, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.6842964887619019, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 0.6614798903465271, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/logs/debug-internal.log b/vb47oelh/wandb/run-20250106_180619-vb47oelh/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..088157eccb809a8af510fd883a957032fa7c818b --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/logs/debug-internal.log @@ -0,0 +1,861 @@ +2025-01-06 18:06:19,073 INFO StreamThr :2098647 [internal.py:wandb_internal():85] W&B internal server running at pid: 2098647, started at: 2025-01-06 18:06:19.072755 +2025-01-06 18:06:19,074 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status +2025-01-06 18:06:19,085 INFO WriterThread:2098647 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/run-vb47oelh.wandb +2025-01-06 18:06:19,085 DEBUG SenderThread:2098647 [sender.py:send():379] send: header +2025-01-06 18:06:19,094 DEBUG SenderThread:2098647 [sender.py:send():379] send: run +2025-01-06 18:06:19,421 INFO SenderThread:2098647 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files +2025-01-06 18:06:19,421 INFO SenderThread:2098647 [sender.py:_start_run_threads():1188] run started: vb47oelh with start time 1736204779.072731 +2025-01-06 18:06:19,438 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 18:06:19,438 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: check_version +2025-01-06 18:06:19,509 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 18:06:19,514 DEBUG HandlerThread:2098647 [system_info.py:__init__():26] System info init +2025-01-06 18:06:19,514 DEBUG HandlerThread:2098647 [system_info.py:__init__():41] System info init done +2025-01-06 18:06:19,514 INFO HandlerThread:2098647 [system_monitor.py:start():194] Starting system monitor +2025-01-06 18:06:19,514 INFO SystemMonitor:2098647 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 18:06:19,514 INFO HandlerThread:2098647 [system_monitor.py:probe():214] Collecting system info +2025-01-06 18:06:19,515 INFO SystemMonitor:2098647 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 18:06:19,516 INFO SystemMonitor:2098647 [interfaces.py:start():188] Started disk monitoring +2025-01-06 18:06:19,517 INFO SystemMonitor:2098647 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 18:06:19,517 INFO SystemMonitor:2098647 [interfaces.py:start():188] Started memory monitoring +2025-01-06 18:06:19,518 INFO SystemMonitor:2098647 [interfaces.py:start():188] Started network monitoring +2025-01-06 18:06:19,526 DEBUG HandlerThread:2098647 [system_info.py:probe():152] Probing system +2025-01-06 18:06:19,528 DEBUG HandlerThread:2098647 [system_info.py:_probe_git():137] Probing git +2025-01-06 18:06:19,533 DEBUG HandlerThread:2098647 [system_info.py:_probe_git():145] Probing git done +2025-01-06 18:06:19,533 DEBUG HandlerThread:2098647 [system_info.py:probe():200] Probing system done +2025-01-06 18:06:19,533 DEBUG HandlerThread:2098647 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T23:06:19.526838', 'startedAt': '2025-01-06T23:06:19.061983', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.11675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 4500.058, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 401.7840385437012}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 18:06:19,533 INFO HandlerThread:2098647 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 18:06:19,533 INFO HandlerThread:2098647 [system_monitor.py:probe():227] Publishing system info +2025-01-06 18:06:19,533 DEBUG HandlerThread:2098647 [system_info.py:_save_code():44] Saving code +2025-01-06 18:06:19,534 DEBUG HandlerThread:2098647 [system_info.py:_save_code():67] Saving code done +2025-01-06 18:06:19,534 DEBUG HandlerThread:2098647 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 18:06:19,596 DEBUG HandlerThread:2098647 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 18:06:19,596 INFO HandlerThread:2098647 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 18:06:19,599 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:06:19,599 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 18:06:19,600 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 18:06:19,600 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 18:06:19,838 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 18:06:19,838 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: python_packages +2025-01-06 18:06:19,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:06:19,839 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:06:19,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:19,842 INFO HandlerThread:2098647 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/tb" + save: true +} +control { + relay_id: "4213779cbc0c4ea28496f8328555bbcc" +} +_info { + stream_id: "vb47oelh" +} + +2025-01-06 18:06:19,842 DEBUG HandlerThread:2098647 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 18:06:19,849 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/28oyl0y4-wandb-metadata.json +2025-01-06 18:06:19,876 INFO wandb-upload_1:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/3qzd1pu5-code/FISH/eval_robot.py +2025-01-06 18:06:19,887 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/pgurgypv-diff.patch +2025-01-06 18:06:19,996 DEBUG SenderThread:2098647 [sender.py:send():379] send: telemetry +2025-01-06 18:06:19,996 DEBUG SenderThread:2098647 [sender.py:send():379] send: tbrecord +2025-01-06 18:06:19,997 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:06:19,997 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 with policy live +2025-01-06 18:06:20,182 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/qm58b42e-events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:06:20,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/requirements.txt +2025-01-06 18:06:20,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/diff.patch +2025-01-06 18:06:20,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/code/FISH/eval_robot.py +2025-01-06 18:06:20,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:06:20,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-metadata.json +2025-01-06 18:06:20,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:06:20,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/code/FISH +2025-01-06 18:06:20,424 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/code +2025-01-06 18:06:20,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:21,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:22,422 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:06:22,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:23,669 DEBUG SenderThread:2098647 [sender.py:send():379] send: config +2025-01-06 18:06:23,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:24,677 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:06:24,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:25,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:26,423 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:06:26,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:27,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:28,424 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:06:28,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:29,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:06:29,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:30,424 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:06:30,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:31,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:32,425 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:06:32,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:33,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:34,838 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:06:34,839 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:06:34,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:34,995 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:06:35,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:36,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:37,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:38,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:39,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:40,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:06:40,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:41,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:42,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:43,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:44,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:45,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:46,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:06:46,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:47,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:48,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:49,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:06:49,839 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:06:49,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:50,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:51,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:51,929 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:06:52,430 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/config.yaml +2025-01-06 18:06:52,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:53,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:54,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:55,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:56,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:57,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:06:57,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:58,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:06:59,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:00,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:01,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:02,433 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:07:02,846 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:03,471 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:03,846 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:04,434 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:07:04,839 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:07:04,839 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:07:04,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:05,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:06,434 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:07:06,846 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:07,846 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:08,846 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:08,928 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:09,847 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:10,847 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:11,847 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:12,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:13,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:14,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:14,849 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:15,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:16,849 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:17,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:18,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:19,518 DEBUG SystemMonitor:2098647 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 18:07:19,519 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:07:19,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:07:19,841 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:07:19,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:19,918 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:20,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:21,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:22,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:23,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:24,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:25,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:25,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:26,850 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:27,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:28,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:29,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:30,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:30,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:31,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:32,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:33,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:34,840 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:07:34,841 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:07:34,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:35,854 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:36,151 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:36,853 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:37,854 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:38,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:39,469 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:07:39,469 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 with policy now +2025-01-06 18:07:39,471 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:07:39,472 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:07:39,472 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:07:39,473 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:07:39,719 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/k9t2peu7-media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 +2025-01-06 18:07:39,854 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:40,443 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:07:40,443 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 +2025-01-06 18:07:40,443 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:07:40,444 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval +2025-01-06 18:07:40,444 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos +2025-01-06 18:07:40,444 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media +2025-01-06 18:07:40,718 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/khbhrbwo-media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 +2025-01-06 18:07:40,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:41,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:41,857 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:42,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:43,854 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:44,445 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:07:44,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:45,854 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:46,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:47,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:47,858 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:48,446 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:07:48,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:49,519 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:07:49,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:07:49,841 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:07:49,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:50,864 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:51,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:52,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:52,995 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:53,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:54,855 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:55,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:56,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:57,449 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:07:57,701 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/fboc6tt8-events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:07:57,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:58,449 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:07:58,677 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:07:58,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:07:59,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:00,450 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:00,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:01,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:02,451 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:02,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:03,856 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:04,327 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:04,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:08:04,842 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:08:04,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:05,857 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:06,857 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:07,857 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:08,858 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:09,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:09,857 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:10,859 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:11,858 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:12,858 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:13,864 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:14,454 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:14,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:14,859 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:15,860 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:16,860 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:17,860 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:18,861 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:19,519 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:08:19,841 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:08:19,842 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:08:19,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:19,983 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:20,861 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:21,862 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:22,457 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:22,861 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:23,862 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:24,862 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:25,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:25,865 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:26,863 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:27,863 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:28,863 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:29,863 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:30,459 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:30,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:30,863 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:31,864 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:32,863 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:33,863 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:34,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:08:34,843 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:08:34,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:35,869 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:36,349 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:36,461 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:36,618 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:08:36,618 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 with policy now +2025-01-06 18:08:36,623 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:08:36,624 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:08:36,624 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:08:36,729 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:08:36,845 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/di80x26n-media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 +2025-01-06 18:08:36,873 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:37,461 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:08:37,462 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 +2025-01-06 18:08:37,462 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval +2025-01-06 18:08:37,729 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/5v4ah568-media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 +2025-01-06 18:08:37,876 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:38,462 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:38,870 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:39,875 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:40,871 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:41,871 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:42,313 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:42,463 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:42,875 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:43,874 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:44,464 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:44,873 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:45,879 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:46,880 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:47,848 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:47,880 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:48,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:49,466 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:08:49,520 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:08:49,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:08:49,843 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:08:49,878 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:50,309 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:08:50,309 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:08:50,309 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:08:50,466 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:50,467 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:08:50,879 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:51,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:52,467 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:52,882 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:53,166 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:53,883 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:54,468 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:54,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:55,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:56,469 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:08:56,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:57,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:58,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:08:58,934 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:08:59,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:00,881 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:01,882 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:02,883 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:03,883 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:04,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:09:04,842 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:09:04,889 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:04,980 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:05,883 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:06,883 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:07,884 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:08,883 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:09,884 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:10,848 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:10,884 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:11,884 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:12,884 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:13,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:14,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:15,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:16,848 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:16,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:17,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:18,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:19,520 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:09:19,842 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:09:19,842 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:09:19,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:20,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:21,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:21,973 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:22,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:23,885 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:24,886 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:25,886 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:26,886 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:27,848 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:27,886 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:28,886 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:29,895 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:30,480 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:09:30,626 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:09:30,626 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 with policy now +2025-01-06 18:09:30,637 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:09:30,638 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:09:30,638 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:09:30,660 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:09:30,856 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/ircl3c9e-media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 +2025-01-06 18:09:30,900 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:31,481 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:09:31,481 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 +2025-01-06 18:09:31,481 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval +2025-01-06 18:09:31,746 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/8t37ch96-media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 +2025-01-06 18:09:31,896 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:32,481 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:09:32,902 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:33,675 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:33,897 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:34,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:09:34,851 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:09:34,897 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:35,910 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:36,482 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:09:36,902 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:37,901 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:38,483 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:09:38,899 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:39,249 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:39,908 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:40,909 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:41,909 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:42,906 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:43,151 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:09:43,151 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:09:43,152 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:09:43,486 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:09:43,486 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:09:43,736 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/c73vqglr-events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:09:43,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:44,487 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:09:44,705 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:44,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:45,916 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:46,908 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:47,909 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:48,489 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:09:48,908 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:49,521 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:09:49,844 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:09:49,844 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:09:49,909 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:49,940 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:50,490 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:09:50,909 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:51,909 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:52,910 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:53,910 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:54,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:55,853 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:09:55,911 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:56,911 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:57,911 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:58,911 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:09:59,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:00,853 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:00,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:01,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:02,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:03,912 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:04,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:10:04,843 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:10:04,913 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:05,913 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:05,994 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:06,913 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:07,913 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:08,913 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:09,913 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:10,914 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:11,854 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:11,914 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:12,914 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:13,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:14,914 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:15,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:16,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:17,854 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:17,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:18,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:19,521 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:10:19,843 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:10:19,843 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:10:19,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:20,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:21,915 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:22,500 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:22,921 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:23,406 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:23,695 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:10:23,695 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 with policy now +2025-01-06 18:10:23,697 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:10:23,698 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:10:23,698 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:10:23,714 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:10:23,923 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:23,931 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/lcary7gx-media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 +2025-01-06 18:10:24,501 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:24,501 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:10:24,501 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 +2025-01-06 18:10:24,502 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval +2025-01-06 18:10:24,764 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/7ysestke-media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 +2025-01-06 18:10:24,924 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:25,935 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:26,923 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:27,926 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:28,437 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:28,502 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:28,923 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:29,928 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:30,926 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:31,927 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:32,504 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:32,926 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:33,933 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:34,233 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:34,851 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:10:34,851 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:10:34,936 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:35,929 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:36,930 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:37,945 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:38,938 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:39,861 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:39,936 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:40,933 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:41,223 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:10:41,223 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:10:41,224 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:10:41,507 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:10:41,507 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:10:41,945 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:42,507 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:42,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:43,945 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:44,508 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:44,945 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:45,858 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:45,954 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:46,509 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:46,946 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:47,946 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:48,509 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:10:48,946 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:49,522 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:10:49,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:10:49,845 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:10:49,946 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:50,946 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:50,996 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:51,946 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:52,946 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:53,947 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:54,947 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:55,947 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:56,857 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:10:56,947 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:57,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:58,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:10:59,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:00,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:01,857 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:01,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:02,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:03,948 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:04,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:11:04,845 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:11:04,949 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:05,949 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:06,927 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:06,949 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:07,949 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:08,949 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:09,949 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:10,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:11,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:12,859 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:12,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:13,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:14,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:15,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:16,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:17,950 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:18,859 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:18,951 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:19,523 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:11:19,845 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:11:19,845 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:11:19,951 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:20,951 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:21,951 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:22,957 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:23,212 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:11:23,212 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 with policy now +2025-01-06 18:11:23,216 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:11:23,216 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:11:23,217 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:23,218 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:23,439 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/3oo7pnjy-media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 +2025-01-06 18:11:23,523 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:11:23,523 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 +2025-01-06 18:11:23,523 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval +2025-01-06 18:11:23,787 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/yc2x18el-media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 +2025-01-06 18:11:23,967 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:24,231 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:24,523 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:24,955 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:25,979 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:26,969 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:27,963 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:28,524 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:28,966 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:29,688 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:29,962 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:30,525 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:30,959 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:31,967 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:32,526 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:11:32,766 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/j460tiwh-events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:11:32,964 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:33,287 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:11:33,287 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:33,287 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:33,351 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 18:11:33,351 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 18:11:33,526 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:11:33,526 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:11:33,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media +2025-01-06 18:11:33,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval +2025-01-06 18:11:33,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table +2025-01-06 18:11:33,540 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:11:33,697 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/qfa78be5-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:11:33,845 INFO wandb-upload_2:2098647 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6qmyzqmn +2025-01-06 18:11:33,923 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 18:11:33,931 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:11:33,956 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:34,227 INFO SenderThread:2098647 [sender.py:send_request_log_artifact():1525] logged artifact run-vb47oelh-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTk5MDU4NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxODI0OA==', 'latestArtifact': None}} +2025-01-06 18:11:34,227 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:11:34,227 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 18:11:34,227 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:11:34,227 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,228 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,228 DEBUG SenderThread:2098647 [sender.py:send():379] send: telemetry +2025-01-06 18:11:34,228 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,232 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,232 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,232 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,232 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,233 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,233 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,233 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,233 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,234 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,234 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,234 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,235 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,235 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,235 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,235 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,236 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,236 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,236 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,236 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,236 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:34,237 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:34,237 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 18:11:34,467 INFO wandb-upload_1:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/o8lfi9aq-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:11:34,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:34,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:11:34,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:11:34,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 18:11:34,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval +2025-01-06 18:11:34,649 INFO wandb-upload_2:2098647 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqycxkxgi +2025-01-06 18:11:34,789 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/ye57dor5-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 18:11:34,846 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:11:34,956 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:35,024 INFO SenderThread:2098647 [sender.py:send_request_log_artifact():1525] logged artifact run-vb47oelh-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTk5MDU5OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxODI0OQ==', 'latestArtifact': None}} +2025-01-06 18:11:35,024 DEBUG SenderThread:2098647 [sender.py:send():379] send: files +2025-01-06 18:11:35,025 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:35,025 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json with policy now +2025-01-06 18:11:35,025 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:11:35,025 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:35,026 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:35,026 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:11:35,271 INFO wandb-upload_1:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/f9qg76k0-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 18:11:35,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:11:35,956 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:36,527 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:36,956 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:37,957 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:38,957 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:39,957 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:40,873 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:40,958 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:41,957 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:42,958 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:43,958 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:44,958 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:45,873 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:45,960 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:46,960 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:47,958 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:48,959 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:49,523 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:11:49,846 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:11:49,846 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:11:49,958 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:11:50,914 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:51,184 DEBUG SenderThread:2098647 [sender.py:send():379] send: exit +2025-01-06 18:11:51,184 INFO SenderThread:2098647 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 18:11:51,184 INFO SenderThread:2098647 [sender.py:send_exit():588] handling runtime: 331 +2025-01-06 18:11:51,184 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:51,184 INFO SenderThread:2098647 [sender.py:send_exit():594] send defer +2025-01-06 18:11:51,184 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:51,185 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 18:11:51,185 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:51,185 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 18:11:51,185 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 1 +2025-01-06 18:11:51,185 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:51,185 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 18:11:51,185 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:51,185 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 18:11:51,185 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 2 +2025-01-06 18:11:51,185 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:51,186 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 18:11:51,186 INFO HandlerThread:2098647 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 18:11:51,186 DEBUG SystemMonitor:2098647 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 18:11:51,186 DEBUG SystemMonitor:2098647 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 18:11:51,186 INFO HandlerThread:2098647 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 18:11:51,187 INFO HandlerThread:2098647 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 18:11:51,221 INFO HandlerThread:2098647 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 18:11:51,221 INFO HandlerThread:2098647 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 18:11:51,221 INFO HandlerThread:2098647 [interfaces.py:finish():200] Joined network monitor +2025-01-06 18:11:51,222 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:51,222 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 18:11:51,222 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 3 +2025-01-06 18:11:51,222 DEBUG SenderThread:2098647 [sender.py:send():379] send: stats +2025-01-06 18:11:51,222 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:51,222 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 18:11:51,222 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:51,222 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 18:11:51,222 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 4 +2025-01-06 18:11:51,223 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:51,223 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 18:11:51,534 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/config.yaml +2025-01-06 18:11:51,534 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:11:52,535 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:57,292 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:11:57,292 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:57,292 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:57,292 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 18:11:57,292 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 5 +2025-01-06 18:11:57,292 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 18:11:57,292 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:11:57,293 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:57,294 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 18:11:57,293 DEBUG SenderThread:2098647 [sender.py:send():379] send: history +2025-01-06 18:11:57,294 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:11:57,294 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:57,295 DEBUG SenderThread:2098647 [sender.py:send():379] send: summary +2025-01-06 18:11:57,298 INFO SenderThread:2098647 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:11:57,299 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:57,299 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 18:11:57,299 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 6 +2025-01-06 18:11:57,299 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:57,299 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 18:11:57,299 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:57,299 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 18:11:57,299 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 7 +2025-01-06 18:11:57,299 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:11:57,299 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:57,299 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 18:11:57,300 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:57,300 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 18:11:57,537 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:11:57,916 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:11:58,349 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 8 +2025-01-06 18:11:58,349 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:11:58,349 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:58,349 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 18:11:58,350 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:58,350 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 18:11:58,350 INFO SenderThread:2098647 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 18:11:58,353 INFO SenderThread:2098647 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 18:11:58,355 INFO SenderThread:2098647 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 18:11:58,370 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 9 +2025-01-06 18:11:58,370 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:58,370 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 18:11:58,370 DEBUG SenderThread:2098647 [sender.py:send():379] send: artifact +2025-01-06 18:11:58,537 INFO Thread-12 :2098647 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:58,810 INFO wandb-upload_2:2098647 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1mspmxl3 +2025-01-06 18:11:58,811 INFO wandb-upload_1:2098647 [upload_job.py:push():85] Skipped uploading /tmp/tmprros7r4x/wandb-job.json +2025-01-06 18:11:58,916 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:11:58,939 INFO wandb-upload_0:2098647 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpntr5kgdh +2025-01-06 18:11:59,520 INFO SenderThread:2098647 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk3NDE3OA==', 'versionIndex': 426}}} +2025-01-06 18:11:59,520 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:59,520 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 18:11:59,520 INFO SenderThread:2098647 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 18:11:59,538 INFO SenderThread:2098647 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files +2025-01-06 18:11:59,538 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/config.yaml config.yaml +2025-01-06 18:11:59,538 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json wandb-summary.json +2025-01-06 18:11:59,539 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:11:59,539 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/requirements.txt requirements.txt +2025-01-06 18:11:59,539 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-metadata.json wandb-metadata.json +2025-01-06 18:11:59,539 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log output.log +2025-01-06 18:11:59,542 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/diff.patch diff.patch +2025-01-06 18:11:59,543 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 18:11:59,543 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 media/videos/eval/2_eval_2_71bebcde765f8093e4db.mp4 +2025-01-06 18:11:59,543 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 media/videos/eval/3_eval_3_9667046e025c1a04f584.mp4 +2025-01-06 18:11:59,543 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 media/videos/eval/0_eval_0_a21f553da3e495971337.mp4 +2025-01-06 18:11:59,543 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 media/videos/eval/4_eval_4_537861e0f91853c73080.mp4 +2025-01-06 18:11:59,543 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 media/videos/eval/1_eval_1_bbe5d527784bc078331e.mp4 +2025-01-06 18:11:59,546 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 18:11:59,546 INFO SenderThread:2098647 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:11:59,546 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 10 +2025-01-06 18:11:59,546 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:11:59,548 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:59,548 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 18:11:59,548 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:59,548 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 18:11:59,548 INFO SenderThread:2098647 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 18:11:59,727 INFO wandb-upload_1:2098647 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/wandb-summary.json +2025-01-06 18:11:59,752 INFO wandb-upload_2:2098647 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/config.yaml +2025-01-06 18:11:59,765 INFO wandb-upload_3:2098647 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/requirements.txt +2025-01-06 18:11:59,780 INFO wandb-upload_4:2098647 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/files/output.log +2025-01-06 18:11:59,785 INFO wandb-upload_0:2098647 [upload_job.py:push():130] Uploaded file /tmp/tmpgjte8_b0wandb/mmn2rth2-events.out.tfevents.1736204779.leonmkim-ROG-Strix-G15CS-G15CS.2098452.0 +2025-01-06 18:11:59,917 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:11:59,917 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:11:59,985 INFO Thread-11 (_thread_body):2098647 [sender.py:transition_state():614] send defer: 11 +2025-01-06 18:11:59,985 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:59,985 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 18:11:59,985 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:59,985 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 18:11:59,985 INFO SenderThread:2098647 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 18:11:59,987 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 12 +2025-01-06 18:11:59,987 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:11:59,987 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 18:11:59,987 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:11:59,987 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 18:11:59,987 INFO SenderThread:2098647 [file_stream.py:finish():601] file stream finish called +2025-01-06 18:12:00,178 INFO SenderThread:2098647 [file_stream.py:finish():605] file stream finish is done +2025-01-06 18:12:00,178 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 13 +2025-01-06 18:12:00,178 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:12:00,178 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 18:12:00,178 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:12:00,178 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 18:12:00,179 INFO SenderThread:2098647 [sender.py:transition_state():614] send defer: 14 +2025-01-06 18:12:00,179 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:12:00,179 INFO HandlerThread:2098647 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 18:12:00,179 DEBUG SenderThread:2098647 [sender.py:send():379] send: final +2025-01-06 18:12:00,179 DEBUG SenderThread:2098647 [sender.py:send():379] send: footer +2025-01-06 18:12:00,179 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: defer +2025-01-06 18:12:00,179 INFO SenderThread:2098647 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 18:12:00,180 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:12:00,180 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:12:00,180 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 18:12:00,180 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: server_info +2025-01-06 18:12:00,183 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:12:00,183 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 18:12:00,184 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 18:12:00,184 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:00,232 DEBUG SenderThread:2098647 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:12:00,232 INFO MainThread:2098647 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 18:12:00,232 INFO MainThread:2098647 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 18:12:00,233 INFO MainThread:2098647 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 18:12:00,233 DEBUG HandlerThread:2098647 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 18:12:00,233 INFO HandlerThread:2098647 [handler.py:finish():882] shutting down handler +2025-01-06 18:12:01,184 INFO WriterThread:2098647 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/run-vb47oelh.wandb +2025-01-06 18:12:01,232 INFO SenderThread:2098647 [sender.py:finish():1615] shutting down sender +2025-01-06 18:12:01,232 INFO SenderThread:2098647 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 18:12:01,232 INFO SenderThread:2098647 [file_pusher.py:join():175] waiting for file pusher diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/logs/debug.log b/vb47oelh/wandb/run-20250106_180619-vb47oelh/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..96fe7c765396d8b14089af599aef60339ccfb510 --- /dev/null +++ b/vb47oelh/wandb/run-20250106_180619-vb47oelh/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Configure stats pid to 2098452 +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/settings +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 18:06:19,065 INFO MainThread:2098452 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 18:06:19,066 INFO MainThread:2098452 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/logs/debug.log +2025-01-06 18:06:19,066 INFO MainThread:2098452 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/wandb/run-20250106_180619-vb47oelh/logs/debug-internal.log +2025-01-06 18:06:19,066 INFO MainThread:2098452 [wandb_init.py:init():569] calling init triggers +2025-01-06 18:06:19,067 INFO MainThread:2098452 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh'} +2025-01-06 18:06:19,067 INFO MainThread:2098452 [wandb_init.py:init():619] starting backend +2025-01-06 18:06:19,067 INFO MainThread:2098452 [wandb_init.py:init():623] setting up manager +2025-01-06 18:06:19,071 INFO MainThread:2098452 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 18:06:19,072 INFO MainThread:2098452 [wandb_init.py:init():631] backend started and connected +2025-01-06 18:06:19,084 INFO MainThread:2098452 [wandb_init.py:init():720] updated telemetry +2025-01-06 18:06:19,092 INFO MainThread:2098452 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 18:06:19,437 INFO MainThread:2098452 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 18:06:19,504 INFO MainThread:2098452 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 18:06:19,504 INFO MainThread:2098452 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 18:06:19,839 INFO MainThread:2098452 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 18:06:19,839 INFO MainThread:2098452 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 18:06:19,840 INFO MainThread:2098452 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 18:06:19,840 INFO MainThread:2098452 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 18:06:19,841 INFO MainThread:2098452 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 18:06:19,841 INFO MainThread:2098452 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/vb47oelh/tb, True +2025-01-06 18:06:23,669 INFO MainThread:2098452 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 18:12:01,233 WARNING MsgRouterThr:2098452 [router.py:message_loop():77] message_loop has been closed diff --git a/vb47oelh/wandb/run-20250106_180619-vb47oelh/run-vb47oelh.wandb b/vb47oelh/wandb/run-20250106_180619-vb47oelh/run-vb47oelh.wandb new file mode 100644 index 0000000000000000000000000000000000000000..6c09c701da78a02bf415f3acb3a7410ff55bce49 Binary files /dev/null and b/vb47oelh/wandb/run-20250106_180619-vb47oelh/run-vb47oelh.wandb differ diff --git a/yjywj48j/episode_rosbags/episode_2_2025-01-07-20-00-16.bag b/yjywj48j/episode_rosbags/episode_2_2025-01-07-20-00-16.bag new file mode 100644 index 0000000000000000000000000000000000000000..1e3b7b8db55b4329b7fd493e4cc3deae19112ae5 --- /dev/null +++ b/yjywj48j/episode_rosbags/episode_2_2025-01-07-20-00-16.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08569f4c6dbfd7e26ee26f8c4cb2bf564d7e94e0772520f444019751c9d1ea53 +size 2610238845 diff --git a/yjywj48j/episode_rosbags/episode_3_2025-01-07-20-01-11.bag b/yjywj48j/episode_rosbags/episode_3_2025-01-07-20-01-11.bag new file mode 100644 index 0000000000000000000000000000000000000000..a9a484b36c5f131142872f37edf1008536a9889e --- /dev/null +++ b/yjywj48j/episode_rosbags/episode_3_2025-01-07-20-01-11.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:829b0631a4a501a33e18c56980686b0058c8eaafef24a78ee4f833f128acf957 +size 2597392777 diff --git a/yjywj48j/episode_rosbags/episode_4_2025-01-07-20-02-04.bag b/yjywj48j/episode_rosbags/episode_4_2025-01-07-20-02-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..14ee8a2275f06c5180bfb6e0cd1980a5eaa20fc8 --- /dev/null +++ b/yjywj48j/episode_rosbags/episode_4_2025-01-07-20-02-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbec760bc5194a0140595e8d894b58f76e0f09af61e62d919e928163d0225cc3 +size 2609762506