diff --git a/.gitattributes b/.gitattributes index d8314634079cd07e1eca3910aef0fa3dae461df9..74b7cacce6c4a64e283d878b335a897daad2a863 100644 --- a/.gitattributes +++ b/.gitattributes @@ -152,3 +152,44 @@ o9qcewu3/episode_rosbags/episode_2_2025-01-07-20-51-45.bag filter=lfs diff=lfs m o9qcewu3/episode_rosbags/episode_3_2025-01-07-20-52-32.bag filter=lfs diff=lfs merge=lfs -text 5jdyg1q2/episode_rosbags/episode_4_2025-01-06-22-12-15.bag filter=lfs diff=lfs merge=lfs -text 5jdyg1q2/episode_rosbags/episode_2_2025-01-06-22-10-54.bag filter=lfs diff=lfs merge=lfs -text +5jdyg1q2/episode_rosbags/episode_0_2025-01-06-22-09-34.bag filter=lfs diff=lfs merge=lfs -text +5jdyg1q2/episode_rosbags/episode_1_2025-01-06-22-10-14.bag filter=lfs diff=lfs merge=lfs -text +9ig9megf/episode_rosbags/episode_2_2025-01-06-22-15-35.bag filter=lfs diff=lfs merge=lfs -text +5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 filter=lfs diff=lfs merge=lfs -text +5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 filter=lfs diff=lfs merge=lfs -text +5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 filter=lfs diff=lfs merge=lfs -text +5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 filter=lfs diff=lfs merge=lfs -text +5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 filter=lfs diff=lfs merge=lfs -text +5jdyg1q2/episode_rosbags/episode_3_2025-01-06-22-11-34.bag filter=lfs diff=lfs merge=lfs -text +5w3hakcj/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5w3hakcj/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5w3hakcj/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5w3hakcj/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5w3hakcj/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +9ig9megf/episode_rosbags/episode_3_2025-01-06-22-16-15.bag filter=lfs diff=lfs merge=lfs -text +9ig9megf/episode_rosbags/episode_0_2025-01-06-22-14-04.bag filter=lfs diff=lfs merge=lfs -text +9ig9megf/episode_rosbags/episode_1_2025-01-06-22-14-52.bag filter=lfs diff=lfs merge=lfs -text +58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 filter=lfs diff=lfs merge=lfs -text +58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 filter=lfs diff=lfs merge=lfs -text +58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 filter=lfs diff=lfs merge=lfs -text +58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 filter=lfs diff=lfs merge=lfs -text +58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 filter=lfs diff=lfs merge=lfs -text +58wmqlmd/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +58wmqlmd/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +58wmqlmd/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +58wmqlmd/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +58wmqlmd/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5w3hakcj/episode_rosbags/episode_1_2025-01-06-17-20-08.bag filter=lfs diff=lfs merge=lfs -text +5w3hakcj/episode_rosbags/episode_2_2025-01-06-17-20-50.bag filter=lfs diff=lfs merge=lfs -text +5w3hakcj/episode_rosbags/episode_0_2025-01-06-17-19-28.bag filter=lfs diff=lfs merge=lfs -text +5w3hakcj/episode_rosbags/episode_4_2025-01-06-17-22-22.bag filter=lfs diff=lfs merge=lfs -text +9fviy2t8/episode_rosbags/episode_1_2025-01-06-17-24-48.bag filter=lfs diff=lfs merge=lfs -text +9fviy2t8/episode_rosbags/episode_2_2025-01-06-17-25-35.bag filter=lfs diff=lfs merge=lfs -text +em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 filter=lfs diff=lfs merge=lfs -text +em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 filter=lfs diff=lfs merge=lfs -text +em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 filter=lfs diff=lfs merge=lfs -text +em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 filter=lfs diff=lfs merge=lfs -text +em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 filter=lfs diff=lfs merge=lfs -text +em58uo70/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +em58uo70/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +em58uo70/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/58wmqlmd/.hydra/config.yaml b/58wmqlmd/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab --- /dev/null +++ b/58wmqlmd/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1009_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/58wmqlmd/.hydra/hydra.yaml b/58wmqlmd/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23d96df68ce1aa1996484031a783e66076b3f69f --- /dev/null +++ b/58wmqlmd/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/58wmqlmd/.hydra/overrides.yaml b/58wmqlmd/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/58wmqlmd/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/58wmqlmd/episode_rosbags/aligned_depth_to_color_K.npy b/58wmqlmd/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/58wmqlmd/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/58wmqlmd/episode_rosbags/cam_tf_world.npy b/58wmqlmd/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/58wmqlmd/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/58wmqlmd/episode_rosbags/color_K.npy b/58wmqlmd/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/58wmqlmd/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/58wmqlmd/episode_rosbags/depth_K.npy b/58wmqlmd/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/58wmqlmd/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/58wmqlmd/eval_robot.log b/58wmqlmd/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..c836d43c9d0a48792ee038c67bf602853706ceae --- /dev/null +++ b/58wmqlmd/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 20:45:05,746][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 20:45:05,749][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/58wmqlmd/eval_video/0_eval.mp4 b/58wmqlmd/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3620be9a41169fb4aaab631be59d3d7405d75d7d --- /dev/null +++ b/58wmqlmd/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0d9ac7b828987714ef06e13c31ecac5a6e32ae2998c6f78f520d67c616e3466 +size 1170020 diff --git a/58wmqlmd/eval_video/1_eval.mp4 b/58wmqlmd/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..78fd16f178a5f20b48ee6f14060e434e46df8b3c --- /dev/null +++ b/58wmqlmd/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3c1331fe3dbbcb17ff394d325162172f3a06d16c2ac007872dde2b668c8b8ac +size 1163004 diff --git a/58wmqlmd/eval_video/2_eval.mp4 b/58wmqlmd/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8aa564180f65c97eff9c5e30cf998a8166014e67 --- /dev/null +++ b/58wmqlmd/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25649ac765ce7bb6dc600a1dc719b6ea424569e80d364f6fd4eedd75d5250073 +size 1164886 diff --git a/58wmqlmd/eval_video/3_eval.mp4 b/58wmqlmd/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..44086dcc8aee392a3e374115339c2c88e3a57d9b --- /dev/null +++ b/58wmqlmd/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f942d3972a955727292ee3d79a9e0252962bafd66e0a8d6df76c01aa6595e5a1 +size 1174046 diff --git a/58wmqlmd/eval_video/4_eval.mp4 b/58wmqlmd/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..45f4f66e5af6d77c59105e70a295dccb72955812 --- /dev/null +++ b/58wmqlmd/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:760e7eee3264b80c008d8a680795904a0720ff25e19882fe873e7555a982ed63 +size 1169990 diff --git a/58wmqlmd/tb/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 b/58wmqlmd/tb/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 new file mode 100644 index 0000000000000000000000000000000000000000..4afae469ac8c59b7711eef23c01f51235aa2ae5b --- /dev/null +++ b/58wmqlmd/tb/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20b42f4311532cc059f8d07d25ea5bcb56d53d0ed96d7b0daece049d778d53a2 +size 6539 diff --git a/58wmqlmd/wandb/debug-internal.log b/58wmqlmd/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..80ce57d266b249b82fcf9adb485c7ac116857236 --- /dev/null +++ b/58wmqlmd/wandb/debug-internal.log @@ -0,0 +1,748 @@ +2025-01-06 20:45:11,469 INFO StreamThr :2114455 [internal.py:wandb_internal():85] W&B internal server running at pid: 2114455, started at: 2025-01-06 20:45:11.469236 +2025-01-06 20:45:11,470 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:45:11,483 INFO WriterThread:2114455 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/run-58wmqlmd.wandb +2025-01-06 20:45:11,483 DEBUG SenderThread:2114455 [sender.py:send():379] send: header +2025-01-06 20:45:11,491 DEBUG SenderThread:2114455 [sender.py:send():379] send: run +2025-01-06 20:45:11,789 INFO SenderThread:2114455 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files +2025-01-06 20:45:11,789 INFO SenderThread:2114455 [sender.py:_start_run_threads():1188] run started: 58wmqlmd with start time 1736214311.470127 +2025-01-06 20:45:11,805 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:45:11,806 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:45:11,862 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:45:11,866 DEBUG HandlerThread:2114455 [system_info.py:__init__():26] System info init +2025-01-06 20:45:11,866 DEBUG HandlerThread:2114455 [system_info.py:__init__():41] System info init done +2025-01-06 20:45:11,866 INFO HandlerThread:2114455 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:45:11,866 INFO SystemMonitor:2114455 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:45:11,867 INFO HandlerThread:2114455 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:45:11,868 INFO SystemMonitor:2114455 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:45:11,868 INFO SystemMonitor:2114455 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:45:11,869 INFO SystemMonitor:2114455 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:45:11,869 INFO SystemMonitor:2114455 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:45:11,870 INFO SystemMonitor:2114455 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:45:11,879 DEBUG HandlerThread:2114455 [system_info.py:probe():152] Probing system +2025-01-06 20:45:11,880 DEBUG HandlerThread:2114455 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:45:11,885 DEBUG HandlerThread:2114455 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:45:11,885 DEBUG HandlerThread:2114455 [system_info.py:probe():200] Probing system done +2025-01-06 20:45:11,885 DEBUG HandlerThread:2114455 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:45:11.879429', 'startedAt': '2025-01-07T01:45:11.461617', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.469375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.904, 'min': 800.0, 'max': 4700.0}, {'current': 4499.924, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.914, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 518.7194671630859}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:45:11,885 INFO HandlerThread:2114455 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:45:11,885 INFO HandlerThread:2114455 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:45:11,889 DEBUG HandlerThread:2114455 [system_info.py:_save_code():44] Saving code +2025-01-06 20:45:11,891 DEBUG HandlerThread:2114455 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:45:11,891 DEBUG HandlerThread:2114455 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:45:11,952 DEBUG HandlerThread:2114455 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:45:11,952 INFO HandlerThread:2114455 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:45:11,956 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:45:11,956 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:45:11,956 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:45:11,956 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:45:12,182 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:45:12,182 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:45:12,182 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:45:12,183 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:45:12,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:12,185 INFO HandlerThread:2114455 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/tb" + save: true +} +control { + relay_id: "e0935560bead4ef6a2b545171ed29f67" +} +_info { + stream_id: "58wmqlmd" +} + +2025-01-06 20:45:12,185 DEBUG HandlerThread:2114455 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:45:12,219 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/k35u63es-code/FISH/eval_robot.py +2025-01-06 20:45:12,241 INFO wandb-upload_2:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/luw4jhex-diff.patch +2025-01-06 20:45:12,241 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/9tpw7i1e-wandb-metadata.json +2025-01-06 20:45:12,286 DEBUG SenderThread:2114455 [sender.py:send():379] send: telemetry +2025-01-06 20:45:12,286 DEBUG SenderThread:2114455 [sender.py:send():379] send: tbrecord +2025-01-06 20:45:12,287 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:45:12,287 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 with policy live +2025-01-06 20:45:12,507 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/gnzzwpnd-events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:45:12,790 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:45:12,791 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/code/FISH/eval_robot.py +2025-01-06 20:45:12,791 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-metadata.json +2025-01-06 20:45:12,791 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:12,791 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/requirements.txt +2025-01-06 20:45:12,791 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/diff.patch +2025-01-06 20:45:12,791 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/code/FISH +2025-01-06 20:45:12,791 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/code +2025-01-06 20:45:13,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:14,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:14,790 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:15,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:16,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:17,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:17,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:18,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:19,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:20,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:21,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:22,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:22,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:23,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:24,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:25,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:26,185 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:27,182 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:45:27,183 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:45:27,225 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:27,305 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:28,154 DEBUG SenderThread:2114455 [sender.py:send():379] send: config +2025-01-06 20:45:28,197 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:29,201 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:30,185 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:30,794 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:31,185 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:32,185 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:32,794 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:32,881 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:33,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:34,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:34,795 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:35,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:36,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:36,796 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:37,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:38,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:38,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:39,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:40,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:41,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:42,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:45:42,183 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:45:42,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:42,797 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:43,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:44,156 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:44,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:44,798 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:44,798 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/config.yaml +2025-01-06 20:45:45,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:46,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:46,799 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:47,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:48,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:49,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:50,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:50,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:51,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:52,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:53,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:54,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:55,185 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:55,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:56,190 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:57,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:45:57,183 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:45:57,225 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:58,190 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:59,190 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:00,197 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:00,275 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:01,191 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:02,191 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:03,191 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:04,191 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:05,191 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:06,191 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:06,192 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:07,192 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:08,193 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:09,194 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:10,202 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:11,202 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:11,870 DEBUG SystemMonitor:2114455 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:46:11,871 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:46:11,872 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:12,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:46:12,183 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:46:12,225 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:13,206 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:13,773 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:46:13,773 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 with policy now +2025-01-06 20:46:13,774 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:46:13,774 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:46:13,774 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:46:13,775 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:46:13,805 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:46:13,806 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 +2025-01-06 20:46:13,806 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos +2025-01-06 20:46:13,806 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media +2025-01-06 20:46:13,806 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval +2025-01-06 20:46:14,028 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/y1o2ohig-media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 +2025-01-06 20:46:14,204 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:14,303 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/7nsgj814-media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 +2025-01-06 20:46:14,806 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:46:15,204 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:16,205 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:17,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:17,205 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:18,205 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:18,807 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:46:19,206 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:20,206 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:20,808 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:46:21,206 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:22,206 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:22,730 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:22,809 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:46:23,049 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/horu78vb-events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:46:23,206 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:24,206 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:24,809 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:46:25,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:26,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:27,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:46:27,186 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:46:27,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:28,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:28,345 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:28,810 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:46:29,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:30,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:31,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:32,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:33,210 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:34,188 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:34,210 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:35,210 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:36,210 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:37,210 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:38,210 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:39,188 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:39,211 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:40,211 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:41,211 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:41,871 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:46:42,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:46:42,186 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:46:42,229 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:43,211 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:44,212 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:44,320 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:45,212 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:46,212 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:47,215 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:48,214 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:49,214 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:50,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:50,215 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:51,215 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:52,215 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:53,215 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:54,215 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:54,819 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:46:54,941 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:46:54,941 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 with policy now +2025-01-06 20:46:54,942 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:46:54,942 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:46:54,942 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:46:54,943 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:46:55,173 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/8e360ud1-media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 +2025-01-06 20:46:55,215 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:55,819 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:46:55,820 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 +2025-01-06 20:46:55,820 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval +2025-01-06 20:46:55,959 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:56,098 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/m1d3ahwf-media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 +2025-01-06 20:46:56,218 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:56,820 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:46:57,195 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:46:57,196 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:46:57,237 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:58,222 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:59,222 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:00,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:00,821 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:01,233 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:01,396 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:02,225 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:02,822 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:03,227 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:04,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:04,823 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:47:05,225 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:05,687 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:47:05,687 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:47:05,687 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:47:05,823 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:47:06,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:06,729 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:06,823 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:07,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:08,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:09,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:10,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:10,825 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:11,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:11,872 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:47:11,872 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:12,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:47:12,186 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:47:12,229 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:13,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:14,227 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:15,227 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:16,227 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:17,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:17,227 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:18,227 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:19,228 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:20,229 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:21,229 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:22,190 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:22,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:23,229 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:24,229 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:25,229 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:26,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:27,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:47:27,187 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:47:27,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:27,254 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:28,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:29,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:30,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:31,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:32,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:33,191 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:33,231 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:34,231 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:35,231 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:36,232 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:36,689 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:47:36,689 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:47:36,690 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 with policy now +2025-01-06 20:47:36,690 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:47:36,690 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:47:36,690 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:47:36,833 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:36,833 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:47:36,834 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 +2025-01-06 20:47:36,834 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval +2025-01-06 20:47:36,926 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/0hvd5p73-media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 +2025-01-06 20:47:37,198 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/pcqbtz2z-media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 +2025-01-06 20:47:37,241 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:38,194 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:38,237 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:38,834 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:39,234 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:40,236 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:41,237 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:41,872 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:47:42,200 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:47:42,200 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:47:42,241 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:42,835 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:43,235 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:44,145 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:44,240 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:44,836 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:45,239 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:46,238 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:47,242 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:48,242 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:48,837 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:47:49,083 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/kwsd77pi-events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:47:49,238 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:49,443 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:47:49,443 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:47:49,443 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:49,443 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:47:49,838 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:47:50,241 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:50,838 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:51,245 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:52,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:53,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:54,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:54,534 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:54,839 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:55,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:56,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:56,840 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:57,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:47:57,187 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:47:57,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:58,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:59,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:00,250 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:00,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:01,387 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:02,387 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:03,387 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:04,387 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:05,387 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:05,534 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:06,387 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:06,843 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:07,388 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:08,388 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:09,388 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:10,388 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:10,581 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:10,845 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:11,388 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:11,873 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:48:12,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:48:12,187 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:48:12,388 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:13,388 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:14,389 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:15,389 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:16,389 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:16,581 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:16,847 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:17,389 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:18,389 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:19,389 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:20,392 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:20,848 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:20,909 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:48:20,909 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 with policy now +2025-01-06 20:48:20,913 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:48:20,913 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:48:20,914 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:48:20,914 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:48:21,134 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/f12j9vht-media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 +2025-01-06 20:48:21,399 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:21,848 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:48:21,849 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 +2025-01-06 20:48:21,849 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval +2025-01-06 20:48:21,936 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:22,137 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/65uaq9mh-media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 +2025-01-06 20:48:22,400 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:22,849 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:23,400 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:23,849 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:24,398 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:25,400 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:26,398 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:26,850 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:27,204 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:27,205 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:48:27,205 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:48:27,403 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:28,402 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:28,851 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:29,401 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:30,401 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:31,401 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:32,409 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:33,203 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:33,406 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:34,407 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:35,404 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:35,854 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:48:36,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:36,499 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:48:36,499 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:48:36,500 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:48:36,854 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:36,854 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:48:37,408 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:38,203 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:38,413 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:38,855 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:39,407 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:40,407 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:40,856 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:41,408 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:41,873 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:48:42,200 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:48:42,200 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:48:42,408 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:43,330 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:43,408 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:44,408 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:45,408 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:46,408 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:47,411 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:48,411 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:49,202 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:49,411 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:50,411 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:51,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:52,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:53,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:54,203 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:54,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:55,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:56,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:57,202 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:48:57,202 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:48:57,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:58,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:59,344 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:59,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:00,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:01,413 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:02,413 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:03,413 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:04,413 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:04,821 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:49:04,865 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:05,413 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:06,418 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:07,425 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:07,768 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:49:07,768 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 with policy now +2025-01-06 20:49:07,768 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:49:07,769 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:49:07,769 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:07,769 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:07,866 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:49:07,866 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 +2025-01-06 20:49:07,866 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval +2025-01-06 20:49:07,996 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/y3bsjbqj-media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 +2025-01-06 20:49:08,268 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/8hu5vf2q-media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 +2025-01-06 20:49:08,421 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:08,866 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:09,423 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:09,867 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:10,424 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:10,797 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:49:11,427 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:11,874 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:49:12,213 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:49:12,213 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:49:12,429 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:12,868 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:13,437 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:14,429 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:15,429 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:16,357 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:49:16,437 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:16,869 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:17,431 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:18,435 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:19,432 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:20,436 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:21,445 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:22,214 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:49:22,441 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:22,555 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:49:22,555 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:22,555 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:22,783 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:49:22,783 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:49:22,871 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:49:22,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:49:22,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:49:22,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media +2025-01-06 20:49:22,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval +2025-01-06 20:49:22,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table +2025-01-06 20:49:22,966 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:49:23,053 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/wljwusm6-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:49:23,121 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/g0nep7sp-events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:49:23,297 INFO wandb-upload_2:2114455 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpox_lvbf5 +2025-01-06 20:49:23,330 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:49:23,339 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:49:23,435 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:23,668 INFO SenderThread:2114455 [sender.py:send_request_log_artifact():1525] logged artifact run-58wmqlmd-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjMyOTE3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY0ODQwMw==', 'latestArtifact': None}} +2025-01-06 20:49:23,669 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:49:23,669 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-06 20:49:23,669 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:49:23,669 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,670 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,670 DEBUG SenderThread:2114455 [sender.py:send():379] send: telemetry +2025-01-06 20:49:23,670 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,670 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,671 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,671 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,671 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,673 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,673 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,673 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,674 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,674 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,674 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,674 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,674 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,675 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,675 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,675 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,675 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,676 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,676 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,676 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,676 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,677 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,677 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:49:23,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:49:23,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:49:23,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:49:23,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval +2025-01-06 20:49:23,907 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/i0u49oyc-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:49:24,062 INFO wandb-upload_2:2114455 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpm48d6fxb +2025-01-06 20:49:24,064 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/nradoyfc-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:49:24,435 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:24,504 INFO SenderThread:2114455 [sender.py:send_request_log_artifact():1525] logged artifact run-58wmqlmd-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjMyOTIwMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY0ODQwNA==', 'latestArtifact': None}} +2025-01-06 20:49:24,504 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:49:24,504 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json with policy now +2025-01-06 20:49:24,505 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:49:24,505 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:24,505 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:24,626 DEBUG SenderThread:2114455 [sender.py:send():379] send: exit +2025-01-06 20:49:24,626 INFO SenderThread:2114455 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:49:24,626 INFO SenderThread:2114455 [sender.py:send_exit():588] handling runtime: 252 +2025-01-06 20:49:24,626 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:24,626 INFO SenderThread:2114455 [sender.py:send_exit():594] send defer +2025-01-06 20:49:24,626 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:24,626 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:49:24,627 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:24,627 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:49:24,627 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:49:24,627 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:24,627 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:49:24,627 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:24,627 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:49:24,627 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:49:24,627 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:24,627 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:49:24,627 INFO HandlerThread:2114455 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:49:24,628 DEBUG SystemMonitor:2114455 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:49:24,628 DEBUG SystemMonitor:2114455 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:49:24,629 INFO HandlerThread:2114455 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:49:24,629 INFO HandlerThread:2114455 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:49:24,680 INFO HandlerThread:2114455 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:49:24,680 INFO HandlerThread:2114455 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:49:24,681 INFO HandlerThread:2114455 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:49:24,681 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:24,681 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:49:24,681 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:49:24,681 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:49:24,681 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:24,681 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:49:24,682 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:24,682 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:49:24,682 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:49:24,682 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:24,682 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:49:24,761 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/54bw3s1j-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:49:24,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:24,873 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:49:26,873 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:31,561 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:49:31,561 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:49:31,562 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:49:31,563 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:31,563 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:49:31,563 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:49:31,563 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:49:31,563 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:49:31,563 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:31,564 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:31,564 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:49:31,564 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:31,564 DEBUG SenderThread:2114455 [sender.py:send():379] send: summary +2025-01-06 20:49:31,568 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:31,568 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:31,568 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:49:31,568 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:49:31,568 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:31,568 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:49:31,568 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:31,568 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:49:31,572 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:49:31,627 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:49:31,695 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:49:31,695 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:49:31,695 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:31,695 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:49:31,695 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:31,695 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:49:31,875 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/config.yaml +2025-01-06 20:49:31,875 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:49:32,627 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:49:32,961 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:49:32,961 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:49:32,962 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:32,962 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:49:32,962 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:32,962 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:49:32,962 INFO SenderThread:2114455 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:49:32,964 INFO SenderThread:2114455 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:49:32,966 INFO SenderThread:2114455 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:49:32,975 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:49:32,975 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:32,976 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:49:32,976 DEBUG SenderThread:2114455 [sender.py:send():379] send: artifact +2025-01-06 20:49:33,181 INFO SenderThread:2114455 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:49:33,181 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:33,181 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:49:33,181 INFO SenderThread:2114455 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:49:33,628 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:49:33,876 INFO SenderThread:2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:33,876 INFO SenderThread:2114455 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files +2025-01-06 20:49:33,877 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/config.yaml config.yaml +2025-01-06 20:49:33,877 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json wandb-summary.json +2025-01-06 20:49:33,877 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/requirements.txt requirements.txt +2025-01-06 20:49:33,877 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:49:33,877 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:49:33,877 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log output.log +2025-01-06 20:49:33,877 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/diff.patch diff.patch +2025-01-06 20:49:33,879 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:49:33,885 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 +2025-01-06 20:49:33,886 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 +2025-01-06 20:49:33,886 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 +2025-01-06 20:49:33,886 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 +2025-01-06 20:49:33,886 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 +2025-01-06 20:49:33,886 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:49:33,886 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:49:33,886 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:49:33,886 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:49:33,886 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:33,886 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:49:33,887 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:33,887 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:49:33,887 INFO SenderThread:2114455 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:49:34,071 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/config.yaml +2025-01-06 20:49:34,076 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/requirements.txt +2025-01-06 20:49:34,097 INFO wandb-upload_2:2114455 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:49:34,139 INFO wandb-upload_4:2114455 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:34,147 INFO wandb-upload_3:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/mkbs4unt-events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:49:34,348 INFO Thread-11 (_thread_body):2114455 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:49:34,348 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:34,348 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:49:34,348 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:34,348 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:49:34,349 INFO SenderThread:2114455 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:49:34,350 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:49:34,350 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:34,350 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:49:34,350 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:34,350 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:49:34,350 INFO SenderThread:2114455 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:49:34,549 INFO SenderThread:2114455 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:49:34,549 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:49:34,549 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:34,549 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:49:34,550 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:34,550 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:49:34,550 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:49:34,550 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:34,550 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:49:34,550 DEBUG SenderThread:2114455 [sender.py:send():379] send: final +2025-01-06 20:49:34,550 DEBUG SenderThread:2114455 [sender.py:send():379] send: footer +2025-01-06 20:49:34,550 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:34,550 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:49:34,551 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:49:34,551 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:49:34,551 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:49:34,551 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:49:34,552 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:49:34,553 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:49:34,554 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:49:34,555 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:34,630 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:49:34,631 INFO MainThread:2114455 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:49:34,631 INFO MainThread:2114455 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:49:34,631 INFO MainThread:2114455 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:49:34,631 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:49:34,632 INFO HandlerThread:2114455 [handler.py:finish():882] shutting down handler +2025-01-06 20:49:35,555 INFO WriterThread:2114455 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/run-58wmqlmd.wandb +2025-01-06 20:49:35,631 INFO SenderThread:2114455 [sender.py:finish():1615] shutting down sender +2025-01-06 20:49:35,631 INFO SenderThread:2114455 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:49:35,631 INFO SenderThread:2114455 [file_pusher.py:join():175] waiting for file pusher diff --git a/58wmqlmd/wandb/debug.log b/58wmqlmd/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..fefed67ed9395e6eadccd1938b288a4a70049b4f --- /dev/null +++ b/58wmqlmd/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Configure stats pid to 2114277 +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/settings +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/logs/debug.log +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/logs/debug-internal.log +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:45:11,464 INFO MainThread:2114277 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd'} +2025-01-06 20:45:11,464 INFO MainThread:2114277 [wandb_init.py:init():619] starting backend +2025-01-06 20:45:11,464 INFO MainThread:2114277 [wandb_init.py:init():623] setting up manager +2025-01-06 20:45:11,468 INFO MainThread:2114277 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:45:11,469 INFO MainThread:2114277 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:45:11,482 INFO MainThread:2114277 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:45:11,488 INFO MainThread:2114277 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:45:11,805 INFO MainThread:2114277 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:45:11,856 INFO MainThread:2114277 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:45:11,856 INFO MainThread:2114277 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:45:12,183 INFO MainThread:2114277 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:45:12,183 INFO MainThread:2114277 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:45:12,183 INFO MainThread:2114277 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:45:12,183 INFO MainThread:2114277 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:45:12,184 INFO MainThread:2114277 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:45:12,184 INFO MainThread:2114277 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/tb, True +2025-01-06 20:45:28,154 INFO MainThread:2114277 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:49:35,633 WARNING MsgRouterThr:2114277 [router.py:message_loop():77] message_loop has been closed diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/code/FISH/eval_robot.py b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/config.yaml b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8073effbd0813437493d4bc6618e3c37bf8b59f --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1009_1' +wandb_notes: + desc: null + value: 1009_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736214311 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/diff.patch b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..43dd1a505c5d25a547a09b1d8e6dc31eb2a6a985 --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..101f9bd 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json new file mode 100644 index 0000000000000000000000000000000000000000..db88f8dce313fe228bb6c4839d50146f71aa6144 --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[20, 20, 1.0, 0.9946215747797684, 1, 0.8453640490079612, 1, 0.865594230519132, 1, 1, 1]]} \ No newline at end of file diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3620be9a41169fb4aaab631be59d3d7405d75d7d --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0d9ac7b828987714ef06e13c31ecac5a6e32ae2998c6f78f520d67c616e3466 +size 1170020 diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..78fd16f178a5f20b48ee6f14060e434e46df8b3c --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3c1331fe3dbbcb17ff394d325162172f3a06d16c2ac007872dde2b668c8b8ac +size 1163004 diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8aa564180f65c97eff9c5e30cf998a8166014e67 --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25649ac765ce7bb6dc600a1dc719b6ea424569e80d364f6fd4eedd75d5250073 +size 1164886 diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..44086dcc8aee392a3e374115339c2c88e3a57d9b --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f942d3972a955727292ee3d79a9e0252962bafd66e0a8d6df76c01aa6595e5a1 +size 1174046 diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..45f4f66e5af6d77c59105e70a295dccb72955812 --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:760e7eee3264b80c008d8a680795904a0720ff25e19882fe873e7555a982ed63 +size 1169990 diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b22165a1804ea02b7f7469d18c6f0e5ce5f1659f --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log @@ -0,0 +1,274 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736214328.155111]: resetting environment +[INFO] [1736214328.157900]: cleared current plan +[INFO] [1736214328.158716]: moving to home +[INFO] [1736214329.160180]: reached home +[INFO] [1736214329.161544]: reset action history +[INFO] [1736214331.870463]: environment reset +Starting episode 0 +[INFO] [1736214331.873024]: resetting environment +[INFO] [1736214331.878425]: cleared current plan +[INFO] [1736214331.879676]: moving to home +[INFO] [1736214332.881377]: reached home +[INFO] [1736214332.882791]: reset action history +[INFO] [1736214335.590911]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736214342.106012]: resetting environment +[INFO] [1736214342.108796]: cleared current plan +[INFO] [1736214342.109581]: moving to home +[INFO] [1736214343.111631]: reached home +[INFO] [1736214343.113249]: reset action history +[INFO] [1736214345.823883]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/episode_rosbags/episode_0_2025-01-06-20-45-45.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2114277.log +started roslaunch server http://158.130.50.37:33463/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2114557] +process[print_text-2]: started with pid [2114581] +process[rosbag_record-3]: started with pid [2114582] +started bagging! +[INFO] [1736214372.817265]: Storing episode... +[rosbag_record-3] killing on exit[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736214373.567697]: Stored episode 1. +[INFO] [1736214373.567992]: Saving video... +[INFO] [1736214373.758639]: Video saved! +Starting episode 1 +[INFO] [1736214373.775249]: resetting environment +[INFO] [1736214373.786171]: cleared current plan +[INFO] [1736214373.786428]: moving to home +[INFO] [1736214377.389283]: reached home +[INFO] [1736214377.390507]: reset action history +[INFO] [1736214380.149427]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736214382.733877]: resetting environment +[INFO] [1736214382.739965]: cleared current plan +[INFO] [1736214382.740163]: moving to home +[INFO] [1736214383.741397]: reached home +[INFO] [1736214383.741801]: reset action history +[INFO] [1736214386.448554]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/episode_rosbags/episode_1_2025-01-06-20-46-26.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2114277.log +started roslaunch server http://158.130.50.37:45477/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2114691] +process[print_text-5]: started with pid [2114715] +process[rosbag_record-6]: started with pid [2114716] +started bagging! +[INFO] [1736214413.455198]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1736214414.196269]: Stored episode 2. +[INFO] [1736214414.196481]: Saving video... +[INFO] [1736214414.923782]: Video saved! +Starting episode 2 +[INFO] [1736214414.943895]: resetting environment +[INFO] [1736214414.958273]: cleared current plan +[INFO] [1736214414.958515]: moving to home +[WARN] [1736214414.804132]: Plan exhausted +[WARN] [1736214414.844613]: Plan exhausted +[WARN] [1736214414.882111]: Plan exhausted +[WARN] [1736214414.919418]: Plan exhausted +[INFO] [1736214418.662060]: reached home +[INFO] [1736214418.667424]: reset action history +[INFO] [1736214421.396161]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736214424.677349]: resetting environment +[INFO] [1736214424.685143]: cleared current plan +[INFO] [1736214424.685453]: moving to home +[INFO] [1736214425.687323]: reached home +[INFO] [1736214425.688133]: reset action history +[INFO] [1736214428.395988]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/episode_rosbags/episode_2_2025-01-06-20-47-08.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2114277.log +started roslaunch server http://158.130.50.37:45349/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2114829] +process[print_text-8]: started with pid [2114830] +process[rosbag_record-9]: started with pid [2114854] +started bagging! +[INFO] [1736214455.363310]: Storing episode... +[print_text-8] killing on exit[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[INFO] [1736214456.145192]: Stored episode 3. +[INFO] [1736214456.150183]: Saving video... +[INFO] [1736214456.642356]: Video saved! +Starting episode 3 +[INFO] [1736214456.678896]: resetting environment +[INFO] [1736214456.699628]: cleared current plan +[INFO] [1736214456.701244]: moving to home +[INFO] [1736214460.309244]: reached home +[INFO] [1736214460.311719]: reset action history +[INFO] [1736214463.098454]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736214468.499737]: resetting environment +[INFO] [1736214468.512151]: cleared current plan +[INFO] [1736214468.515230]: moving to home +[INFO] [1736214469.518773]: reached home +[INFO] [1736214469.520307]: reset action history +[INFO] [1736214472.412513]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/episode_rosbags/episode_3_2025-01-06-20-47-52.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2114277.log +started roslaunch server http://158.130.50.37:44325/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2114985] +process[print_text-11]: started with pid [2114986] +process[rosbag_record-12]: started with pid [2114987] +started bagging! +For topic gripper_width: timestamp difference is -103699872 for nearest: 1736214474133264592 - target: 1736214474236964464 at idx 26 +For topic gripper_width: timestamp difference is -103580997 for nearest: 1736214474333225920 - target: 1736214474436806917 at idx 27 +For topic gripper_width: timestamp difference is 105232041 for nearest: 1736214485533226054 - target: 1736214485427994013 at idx 179 +For topic gripper_width: timestamp difference is 108440271 for nearest: 1736214489533236137 - target: 1736214489424795866 at idx 235 +For topic gripper_width: timestamp difference is 112477576 for nearest: 1736214494533228194 - target: 1736214494420750618 at idx 249 +[INFO] [1736214499.540979]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736214500.290054]: Stored episode 4. +[INFO] [1736214500.305401]: Saving video... +[INFO] [1736214500.865533]: Video saved! +Starting episode 4 +[INFO] [1736214500.918716]: resetting environment +[INFO] [1736214500.932265]: cleared current plan +[INFO] [1736214500.933658]: moving to home +[WARN] [1736214500.881989]: Plan exhausted +[WARN] [1736214500.920216]: Plan exhausted +[INFO] [1736214504.640216]: reached home +[INFO] [1736214504.643129]: reset action history +[INFO] [1736214507.431746]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736214515.543439]: resetting environment +[INFO] [1736214515.556370]: cleared current plan +[INFO] [1736214515.559312]: moving to home +[INFO] [1736214516.565203]: reached home +[INFO] [1736214516.571810]: reset action history +[INFO] [1736214519.347628]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/episode_rosbags/episode_4_2025-01-06-20-48-39.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2114277.log +started roslaunch server http://158.130.50.37:36165/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2115098] +process[print_text-14]: started with pid [2115099] +process[rosbag_record-15]: started with pid [2115100] +started bagging! +For topic gripper_width: timestamp difference is -215109040 for nearest: 1736214542433271955 - target: 1736214542648380995 at idx 249 +[INFO] [1736214546.351594]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1736214547.227752]: Stored episode 5. +[INFO] [1736214547.233466]: Saving video... +[INFO] [1736214547.720211]: Video saved! +[INFO] [1736214547.769995]: resetting environment +[INFO] [1736214547.783609]: cleared current plan +[INFO] [1736214547.792595]: moving to home +[WARN] [1736214547.692914]: Plan exhausted +[WARN] [1736214547.727910]: Plan exhausted +[WARN] [1736214547.759794]: Plan exhausted +[INFO] [1736214551.520930]: reached home +[INFO] [1736214551.525765]: reset action history +[INFO] [1736214554.311278]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/requirements.txt b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-metadata.json b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5dff8e9a6a7b192f3bea70e6c0c632859295135c --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T01:45:11.879429", + "startedAt": "2025-01-07T01:45:11.461617", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.469375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.904, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.924, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.914, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 518.7194671630859 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ed0add117744a76159278122135acad482394d2e --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "d0d9ac7b828987714ef06e13c31ecac5a6e32ae2998c6f78f520d67c616e3466", "size": 1170020, "path": "media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4"}, "global_step": 664, "_timestamp": 1736214563.2325444, "_runtime": 251.76241731643677, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "a3c1331fe3dbbcb17ff394d325162172f3a06d16c2ac007872dde2b668c8b8ac", "size": 1163004, "path": "media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4"}, "eval/num_success": 5.0, "episode": 20.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "25649ac765ce7bb6dc600a1dc719b6ea424569e80d364f6fd4eedd75d5250073", "size": 1164886, "path": "media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "f942d3972a955727292ee3d79a9e0252962bafd66e0a8d6df76c01aa6595e5a1", "size": 1174046, "path": "media/videos/eval/3_eval_3_f942d3972a955727292e.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "760e7eee3264b80c008d8a680795904a0720ff25e19882fe873e7555a982ed63", "size": 1169990, "path": "media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://ohtyi697zi7i8t8q2adyg0rwsruhz0xpx0iifmnt5hvdy05h7lyr4jcrigo72qslhedgfrx6n82lbbagm6nykdra282nqfgmdy0puy3x2kxoibn4uh3vtuq43i6b0uso/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://hklytw4ws4d4f4d7u3dfli1qi7orwj4znw32u0aezigb4tltnjen4uxk0zhyiehek1kym3thh5724igfaigavtu1ges80g2xqk6wwuu28zkqngahqekl0j66xhug3w0d:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 20, "total_num_episodes": 20, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9946215747797684, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.8453640490079612, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.865594230519132, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "c68d11fd8ad6b790b7a949bb003b7716c3f8e083e007b71762bd414593b8472d", "size": 419, "artifact_path": "wandb-client-artifact://fby84aw30b4rh72zlu527mscqgcajbtb8ktz87qb5siqit2w0w64mhjqwnm9p0redioazxcru29k1446u96pju6odopi6jgigli9kslx7noy4b4edfiv5gdct9nbl1sa/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://xv4bxbgqhbmqeqmr3lb2mokzqqaoo5vsqlsk01dqwfoj9t8fnb55vmyfl5f7brvspl23o64kereavq0d2j7i43k3yl6sc1vmjr8p0i6fau0yx98p3t21e1cay3gu7y0b:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 252}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9946215748786926, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.8453640341758728, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.865594208240509, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/logs/debug-internal.log b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..80ce57d266b249b82fcf9adb485c7ac116857236 --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/logs/debug-internal.log @@ -0,0 +1,748 @@ +2025-01-06 20:45:11,469 INFO StreamThr :2114455 [internal.py:wandb_internal():85] W&B internal server running at pid: 2114455, started at: 2025-01-06 20:45:11.469236 +2025-01-06 20:45:11,470 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:45:11,483 INFO WriterThread:2114455 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/run-58wmqlmd.wandb +2025-01-06 20:45:11,483 DEBUG SenderThread:2114455 [sender.py:send():379] send: header +2025-01-06 20:45:11,491 DEBUG SenderThread:2114455 [sender.py:send():379] send: run +2025-01-06 20:45:11,789 INFO SenderThread:2114455 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files +2025-01-06 20:45:11,789 INFO SenderThread:2114455 [sender.py:_start_run_threads():1188] run started: 58wmqlmd with start time 1736214311.470127 +2025-01-06 20:45:11,805 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:45:11,806 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:45:11,862 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:45:11,866 DEBUG HandlerThread:2114455 [system_info.py:__init__():26] System info init +2025-01-06 20:45:11,866 DEBUG HandlerThread:2114455 [system_info.py:__init__():41] System info init done +2025-01-06 20:45:11,866 INFO HandlerThread:2114455 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:45:11,866 INFO SystemMonitor:2114455 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:45:11,867 INFO HandlerThread:2114455 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:45:11,868 INFO SystemMonitor:2114455 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:45:11,868 INFO SystemMonitor:2114455 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:45:11,869 INFO SystemMonitor:2114455 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:45:11,869 INFO SystemMonitor:2114455 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:45:11,870 INFO SystemMonitor:2114455 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:45:11,879 DEBUG HandlerThread:2114455 [system_info.py:probe():152] Probing system +2025-01-06 20:45:11,880 DEBUG HandlerThread:2114455 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:45:11,885 DEBUG HandlerThread:2114455 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:45:11,885 DEBUG HandlerThread:2114455 [system_info.py:probe():200] Probing system done +2025-01-06 20:45:11,885 DEBUG HandlerThread:2114455 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:45:11.879429', 'startedAt': '2025-01-07T01:45:11.461617', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.469375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.904, 'min': 800.0, 'max': 4700.0}, {'current': 4499.924, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.914, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 518.7194671630859}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:45:11,885 INFO HandlerThread:2114455 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:45:11,885 INFO HandlerThread:2114455 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:45:11,889 DEBUG HandlerThread:2114455 [system_info.py:_save_code():44] Saving code +2025-01-06 20:45:11,891 DEBUG HandlerThread:2114455 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:45:11,891 DEBUG HandlerThread:2114455 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:45:11,952 DEBUG HandlerThread:2114455 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:45:11,952 INFO HandlerThread:2114455 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:45:11,956 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:45:11,956 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:45:11,956 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:45:11,956 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:45:12,182 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:45:12,182 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:45:12,182 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:45:12,183 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:45:12,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:12,185 INFO HandlerThread:2114455 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/tb" + save: true +} +control { + relay_id: "e0935560bead4ef6a2b545171ed29f67" +} +_info { + stream_id: "58wmqlmd" +} + +2025-01-06 20:45:12,185 DEBUG HandlerThread:2114455 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:45:12,219 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/k35u63es-code/FISH/eval_robot.py +2025-01-06 20:45:12,241 INFO wandb-upload_2:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/luw4jhex-diff.patch +2025-01-06 20:45:12,241 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/9tpw7i1e-wandb-metadata.json +2025-01-06 20:45:12,286 DEBUG SenderThread:2114455 [sender.py:send():379] send: telemetry +2025-01-06 20:45:12,286 DEBUG SenderThread:2114455 [sender.py:send():379] send: tbrecord +2025-01-06 20:45:12,287 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:45:12,287 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 with policy live +2025-01-06 20:45:12,507 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/gnzzwpnd-events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:45:12,790 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:45:12,791 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/code/FISH/eval_robot.py +2025-01-06 20:45:12,791 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-metadata.json +2025-01-06 20:45:12,791 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:12,791 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/requirements.txt +2025-01-06 20:45:12,791 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/diff.patch +2025-01-06 20:45:12,791 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/code/FISH +2025-01-06 20:45:12,791 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/code +2025-01-06 20:45:13,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:14,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:14,790 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:15,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:16,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:17,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:17,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:18,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:19,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:20,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:21,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:22,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:22,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:23,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:24,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:25,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:26,185 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:27,182 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:45:27,183 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:45:27,225 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:27,305 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:28,154 DEBUG SenderThread:2114455 [sender.py:send():379] send: config +2025-01-06 20:45:28,197 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:29,201 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:30,185 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:30,794 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:31,185 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:32,185 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:32,794 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:32,881 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:33,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:34,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:34,795 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:35,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:36,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:36,796 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:37,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:38,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:38,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:39,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:40,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:41,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:42,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:45:42,183 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:45:42,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:42,797 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:43,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:44,156 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:44,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:44,798 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:44,798 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/config.yaml +2025-01-06 20:45:45,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:46,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:46,799 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:45:47,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:48,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:49,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:50,184 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:50,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:51,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:52,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:53,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:54,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:55,185 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:45:55,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:56,190 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:57,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:45:57,183 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:45:57,225 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:58,190 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:45:59,190 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:00,197 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:00,275 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:01,191 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:02,191 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:03,191 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:04,191 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:05,191 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:06,191 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:06,192 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:07,192 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:08,193 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:09,194 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:10,202 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:11,202 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:11,870 DEBUG SystemMonitor:2114455 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:46:11,871 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:46:11,872 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:12,183 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:46:12,183 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:46:12,225 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:13,206 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:13,773 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:46:13,773 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 with policy now +2025-01-06 20:46:13,774 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:46:13,774 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:46:13,774 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:46:13,775 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:46:13,805 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:46:13,806 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 +2025-01-06 20:46:13,806 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos +2025-01-06 20:46:13,806 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media +2025-01-06 20:46:13,806 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval +2025-01-06 20:46:14,028 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/y1o2ohig-media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 +2025-01-06 20:46:14,204 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:14,303 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/7nsgj814-media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 +2025-01-06 20:46:14,806 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:46:15,204 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:16,205 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:17,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:17,205 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:18,205 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:18,807 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:46:19,206 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:20,206 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:20,808 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:46:21,206 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:22,206 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:22,730 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:22,809 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:46:23,049 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/horu78vb-events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:46:23,206 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:24,206 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:24,809 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:46:25,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:26,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:27,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:46:27,186 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:46:27,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:28,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:28,345 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:28,810 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:46:29,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:30,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:31,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:32,207 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:33,210 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:34,188 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:34,210 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:35,210 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:36,210 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:37,210 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:38,210 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:39,188 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:39,211 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:40,211 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:41,211 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:41,871 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:46:42,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:46:42,186 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:46:42,229 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:43,211 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:44,212 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:44,320 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:45,212 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:46,212 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:47,215 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:48,214 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:49,214 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:50,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:50,215 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:51,215 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:52,215 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:53,215 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:54,215 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:54,819 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:46:54,941 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:46:54,941 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 with policy now +2025-01-06 20:46:54,942 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:46:54,942 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:46:54,942 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:46:54,943 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:46:55,173 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/8e360ud1-media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 +2025-01-06 20:46:55,215 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:55,819 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:46:55,820 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 +2025-01-06 20:46:55,820 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval +2025-01-06 20:46:55,959 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:46:56,098 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/m1d3ahwf-media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 +2025-01-06 20:46:56,218 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:56,820 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:46:57,195 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:46:57,196 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:46:57,237 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:58,222 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:46:59,222 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:00,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:00,821 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:01,233 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:01,396 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:02,225 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:02,822 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:03,227 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:04,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:04,823 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:47:05,225 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:05,687 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:47:05,687 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:47:05,687 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:47:05,823 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:47:06,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:06,729 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:06,823 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:07,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:08,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:09,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:10,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:10,825 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:11,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:11,872 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:47:11,872 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:12,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:47:12,186 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:47:12,229 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:13,226 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:14,227 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:15,227 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:16,227 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:17,189 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:17,227 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:18,227 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:19,228 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:20,229 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:21,229 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:22,190 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:22,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:23,229 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:24,229 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:25,229 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:26,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:27,186 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:47:27,187 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:47:27,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:27,254 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:28,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:29,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:30,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:31,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:32,230 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:33,191 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:33,231 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:34,231 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:35,231 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:36,232 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:36,689 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:47:36,689 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:47:36,690 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 with policy now +2025-01-06 20:47:36,690 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:47:36,690 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:47:36,690 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:47:36,833 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:36,833 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:47:36,834 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 +2025-01-06 20:47:36,834 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval +2025-01-06 20:47:36,926 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/0hvd5p73-media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 +2025-01-06 20:47:37,198 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/pcqbtz2z-media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 +2025-01-06 20:47:37,241 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:38,194 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:38,237 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:38,834 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:39,234 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:40,236 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:41,237 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:41,872 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:47:42,200 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:47:42,200 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:47:42,241 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:42,835 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:43,235 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:44,145 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:44,240 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:44,836 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:45,239 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:46,238 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:47,242 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:48,242 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:48,837 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:47:49,083 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/kwsd77pi-events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:47:49,238 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:49,443 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:47:49,443 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:47:49,443 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:49,443 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:47:49,838 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:47:50,241 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:50,838 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:51,245 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:52,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:53,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:54,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:54,534 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:47:54,839 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:55,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:56,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:56,840 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:47:57,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:47:57,187 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:47:57,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:58,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:47:59,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:00,250 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:00,386 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:01,387 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:02,387 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:03,387 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:04,387 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:05,387 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:05,534 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:06,387 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:06,843 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:07,388 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:08,388 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:09,388 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:10,388 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:10,581 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:10,845 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:11,388 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:11,873 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:48:12,187 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:48:12,187 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:48:12,388 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:13,388 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:14,389 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:15,389 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:16,389 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:16,581 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:16,847 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:17,389 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:18,389 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:19,389 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:20,392 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:20,848 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:20,909 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:48:20,909 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 with policy now +2025-01-06 20:48:20,913 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:48:20,913 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:48:20,914 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:48:20,914 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:48:21,134 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/f12j9vht-media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 +2025-01-06 20:48:21,399 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:21,848 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:48:21,849 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 +2025-01-06 20:48:21,849 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval +2025-01-06 20:48:21,936 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:22,137 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/65uaq9mh-media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 +2025-01-06 20:48:22,400 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:22,849 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:23,400 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:23,849 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:24,398 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:25,400 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:26,398 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:26,850 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:27,204 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:27,205 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:48:27,205 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:48:27,403 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:28,402 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:28,851 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:29,401 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:30,401 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:31,401 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:32,409 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:33,203 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:33,406 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:34,407 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:35,404 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:35,854 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:48:36,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:36,499 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:48:36,499 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:48:36,500 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:48:36,854 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:36,854 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:48:37,408 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:38,203 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:38,413 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:38,855 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:39,407 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:40,407 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:40,856 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:48:41,408 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:41,873 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:48:42,200 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:48:42,200 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:48:42,408 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:43,330 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:43,408 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:44,408 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:45,408 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:46,408 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:47,411 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:48,411 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:49,202 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:49,411 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:50,411 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:51,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:52,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:53,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:54,203 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:54,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:55,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:56,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:57,202 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:48:57,202 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:48:57,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:58,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:48:59,344 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:48:59,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:00,412 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:01,413 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:02,413 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:03,413 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:04,413 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:04,821 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:49:04,865 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:05,413 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:06,418 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:07,425 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:07,768 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:49:07,768 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 with policy now +2025-01-06 20:49:07,768 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:49:07,769 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:49:07,769 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:07,769 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:07,866 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:49:07,866 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 +2025-01-06 20:49:07,866 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval +2025-01-06 20:49:07,996 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/y3bsjbqj-media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 +2025-01-06 20:49:08,268 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/8hu5vf2q-media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 +2025-01-06 20:49:08,421 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:08,866 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:09,423 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:09,867 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:10,424 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:10,797 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:49:11,427 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:11,874 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:49:12,213 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:49:12,213 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:49:12,429 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:12,868 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:13,437 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:14,429 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:15,429 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:16,357 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:49:16,437 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:16,869 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:17,431 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:18,435 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:19,432 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:20,436 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:21,445 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:22,214 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:49:22,441 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:22,555 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:49:22,555 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:22,555 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:22,783 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:49:22,783 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:49:22,871 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:49:22,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:49:22,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:49:22,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media +2025-01-06 20:49:22,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval +2025-01-06 20:49:22,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table +2025-01-06 20:49:22,966 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:49:23,053 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/wljwusm6-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:49:23,121 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/g0nep7sp-events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:49:23,297 INFO wandb-upload_2:2114455 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpox_lvbf5 +2025-01-06 20:49:23,330 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:49:23,339 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:49:23,435 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:23,668 INFO SenderThread:2114455 [sender.py:send_request_log_artifact():1525] logged artifact run-58wmqlmd-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjMyOTE3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY0ODQwMw==', 'latestArtifact': None}} +2025-01-06 20:49:23,669 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:49:23,669 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-06 20:49:23,669 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:49:23,669 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,670 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,670 DEBUG SenderThread:2114455 [sender.py:send():379] send: telemetry +2025-01-06 20:49:23,670 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,670 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,671 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,671 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,671 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,673 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,673 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,673 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,674 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,674 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,674 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,674 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,674 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,675 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,675 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,675 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,675 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,676 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,676 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,676 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,676 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:23,677 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:23,677 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:49:23,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:49:23,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:49:23,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:49:23,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval +2025-01-06 20:49:23,907 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/i0u49oyc-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:49:24,062 INFO wandb-upload_2:2114455 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpm48d6fxb +2025-01-06 20:49:24,064 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/nradoyfc-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:49:24,435 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:24,504 INFO SenderThread:2114455 [sender.py:send_request_log_artifact():1525] logged artifact run-58wmqlmd-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjMyOTIwMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY0ODQwNA==', 'latestArtifact': None}} +2025-01-06 20:49:24,504 DEBUG SenderThread:2114455 [sender.py:send():379] send: files +2025-01-06 20:49:24,504 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json with policy now +2025-01-06 20:49:24,505 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:49:24,505 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:24,505 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:24,626 DEBUG SenderThread:2114455 [sender.py:send():379] send: exit +2025-01-06 20:49:24,626 INFO SenderThread:2114455 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:49:24,626 INFO SenderThread:2114455 [sender.py:send_exit():588] handling runtime: 252 +2025-01-06 20:49:24,626 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:24,626 INFO SenderThread:2114455 [sender.py:send_exit():594] send defer +2025-01-06 20:49:24,626 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:24,626 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:49:24,627 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:24,627 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:49:24,627 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:49:24,627 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:24,627 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:49:24,627 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:24,627 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:49:24,627 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:49:24,627 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:24,627 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:49:24,627 INFO HandlerThread:2114455 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:49:24,628 DEBUG SystemMonitor:2114455 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:49:24,628 DEBUG SystemMonitor:2114455 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:49:24,629 INFO HandlerThread:2114455 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:49:24,629 INFO HandlerThread:2114455 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:49:24,680 INFO HandlerThread:2114455 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:49:24,680 INFO HandlerThread:2114455 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:49:24,681 INFO HandlerThread:2114455 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:49:24,681 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:24,681 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:49:24,681 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:49:24,681 DEBUG SenderThread:2114455 [sender.py:send():379] send: stats +2025-01-06 20:49:24,681 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:24,681 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:49:24,682 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:24,682 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:49:24,682 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:49:24,682 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:24,682 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:49:24,761 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/54bw3s1j-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:49:24,872 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:24,873 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:49:26,873 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:31,561 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:49:31,561 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:49:31,562 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:49:31,563 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:31,563 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:49:31,563 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:49:31,563 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:49:31,563 DEBUG SenderThread:2114455 [sender.py:send():379] send: history +2025-01-06 20:49:31,563 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:49:31,564 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:31,564 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:49:31,564 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:31,564 DEBUG SenderThread:2114455 [sender.py:send():379] send: summary +2025-01-06 20:49:31,568 INFO SenderThread:2114455 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:49:31,568 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:31,568 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:49:31,568 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:49:31,568 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:31,568 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:49:31,568 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:31,568 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:49:31,572 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:49:31,627 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:49:31,695 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:49:31,695 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:49:31,695 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:31,695 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:49:31,695 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:31,695 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:49:31,875 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/config.yaml +2025-01-06 20:49:31,875 INFO Thread-12 :2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:49:32,627 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:49:32,961 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:49:32,961 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:49:32,962 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:32,962 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:49:32,962 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:32,962 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:49:32,962 INFO SenderThread:2114455 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:49:32,964 INFO SenderThread:2114455 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:49:32,966 INFO SenderThread:2114455 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:49:32,975 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:49:32,975 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:32,976 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:49:32,976 DEBUG SenderThread:2114455 [sender.py:send():379] send: artifact +2025-01-06 20:49:33,181 INFO SenderThread:2114455 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:49:33,181 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:33,181 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:49:33,181 INFO SenderThread:2114455 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:49:33,628 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:49:33,876 INFO SenderThread:2114455 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:33,876 INFO SenderThread:2114455 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files +2025-01-06 20:49:33,877 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/config.yaml config.yaml +2025-01-06 20:49:33,877 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json wandb-summary.json +2025-01-06 20:49:33,877 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/requirements.txt requirements.txt +2025-01-06 20:49:33,877 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:49:33,877 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:49:33,877 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log output.log +2025-01-06 20:49:33,877 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/diff.patch diff.patch +2025-01-06 20:49:33,879 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:49:33,885 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 media/videos/eval/4_eval_4_760e7eee3264b80c008d.mp4 +2025-01-06 20:49:33,886 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 media/videos/eval/0_eval_0_d0d9ac7b828987714ef0.mp4 +2025-01-06 20:49:33,886 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 media/videos/eval/2_eval_2_25649ac765ce7bb6dc60.mp4 +2025-01-06 20:49:33,886 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 media/videos/eval/3_eval_3_f942d3972a955727292e.mp4 +2025-01-06 20:49:33,886 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 media/videos/eval/1_eval_1_a3c1331fe3dbbcb17ff3.mp4 +2025-01-06 20:49:33,886 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:49:33,886 INFO SenderThread:2114455 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:49:33,886 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:49:33,886 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:49:33,886 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:33,886 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:49:33,887 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:33,887 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:49:33,887 INFO SenderThread:2114455 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:49:34,071 INFO wandb-upload_0:2114455 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/config.yaml +2025-01-06 20:49:34,076 INFO wandb-upload_1:2114455 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/requirements.txt +2025-01-06 20:49:34,097 INFO wandb-upload_2:2114455 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/wandb-summary.json +2025-01-06 20:49:34,139 INFO wandb-upload_4:2114455 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/files/output.log +2025-01-06 20:49:34,147 INFO wandb-upload_3:2114455 [upload_job.py:push():130] Uploaded file /tmp/tmpzv1tdaddwandb/mkbs4unt-events.out.tfevents.1736214312.leonmkim-ROG-Strix-G15CS-G15CS.2114277.0 +2025-01-06 20:49:34,348 INFO Thread-11 (_thread_body):2114455 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:49:34,348 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:34,348 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:49:34,348 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:34,348 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:49:34,349 INFO SenderThread:2114455 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:49:34,350 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:49:34,350 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:34,350 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:49:34,350 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:34,350 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:49:34,350 INFO SenderThread:2114455 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:49:34,549 INFO SenderThread:2114455 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:49:34,549 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:49:34,549 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:34,549 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:49:34,550 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:34,550 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:49:34,550 INFO SenderThread:2114455 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:49:34,550 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:49:34,550 INFO HandlerThread:2114455 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:49:34,550 DEBUG SenderThread:2114455 [sender.py:send():379] send: final +2025-01-06 20:49:34,550 DEBUG SenderThread:2114455 [sender.py:send():379] send: footer +2025-01-06 20:49:34,550 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: defer +2025-01-06 20:49:34,550 INFO SenderThread:2114455 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:49:34,551 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:49:34,551 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:49:34,551 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:49:34,551 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:49:34,552 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:49:34,553 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:49:34,554 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:49:34,555 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:49:34,630 DEBUG SenderThread:2114455 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:49:34,631 INFO MainThread:2114455 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:49:34,631 INFO MainThread:2114455 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:49:34,631 INFO MainThread:2114455 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:49:34,631 DEBUG HandlerThread:2114455 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:49:34,632 INFO HandlerThread:2114455 [handler.py:finish():882] shutting down handler +2025-01-06 20:49:35,555 INFO WriterThread:2114455 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/run-58wmqlmd.wandb +2025-01-06 20:49:35,631 INFO SenderThread:2114455 [sender.py:finish():1615] shutting down sender +2025-01-06 20:49:35,631 INFO SenderThread:2114455 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:49:35,631 INFO SenderThread:2114455 [file_pusher.py:join():175] waiting for file pusher diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/logs/debug.log b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..fefed67ed9395e6eadccd1938b288a4a70049b4f --- /dev/null +++ b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Configure stats pid to 2114277 +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/settings +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/logs/debug.log +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/logs/debug-internal.log +2025-01-06 20:45:11,463 INFO MainThread:2114277 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:45:11,464 INFO MainThread:2114277 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd'} +2025-01-06 20:45:11,464 INFO MainThread:2114277 [wandb_init.py:init():619] starting backend +2025-01-06 20:45:11,464 INFO MainThread:2114277 [wandb_init.py:init():623] setting up manager +2025-01-06 20:45:11,468 INFO MainThread:2114277 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:45:11,469 INFO MainThread:2114277 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:45:11,482 INFO MainThread:2114277 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:45:11,488 INFO MainThread:2114277 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:45:11,805 INFO MainThread:2114277 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:45:11,856 INFO MainThread:2114277 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:45:11,856 INFO MainThread:2114277 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:45:12,183 INFO MainThread:2114277 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:45:12,183 INFO MainThread:2114277 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:45:12,183 INFO MainThread:2114277 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:45:12,183 INFO MainThread:2114277 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:45:12,184 INFO MainThread:2114277 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:45:12,184 INFO MainThread:2114277 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/58wmqlmd/tb, True +2025-01-06 20:45:28,154 INFO MainThread:2114277 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:49:35,633 WARNING MsgRouterThr:2114277 [router.py:message_loop():77] message_loop has been closed diff --git a/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/run-58wmqlmd.wandb b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/run-58wmqlmd.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ee3dcd0966bffb6fd15d5a4914d2c13533f8d7d4 Binary files /dev/null and b/58wmqlmd/wandb/run-20250106_204511-58wmqlmd/run-58wmqlmd.wandb differ diff --git a/5jdyg1q2/episode_rosbags/episode_0_2025-01-06-22-09-34.bag b/5jdyg1q2/episode_rosbags/episode_0_2025-01-06-22-09-34.bag new file mode 100644 index 0000000000000000000000000000000000000000..7272f500fc683c00e70bcf77b5eac4e68884dbab --- /dev/null +++ b/5jdyg1q2/episode_rosbags/episode_0_2025-01-06-22-09-34.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2119e12cbfec599c56aa2a0ce0f97f98a0859f6bc8e8071f392249eeb6e905e0 +size 1408882178 diff --git a/5jdyg1q2/episode_rosbags/episode_1_2025-01-06-22-10-14.bag b/5jdyg1q2/episode_rosbags/episode_1_2025-01-06-22-10-14.bag new file mode 100644 index 0000000000000000000000000000000000000000..d3560bd1b09dda7276f69aed8fb0eead7aacbf9a --- /dev/null +++ b/5jdyg1q2/episode_rosbags/episode_1_2025-01-06-22-10-14.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:313a46a9320f288a7c9ccecc888fdb927c3aa3404ed9b1a5d1d9f293aa90a055 +size 1413896398 diff --git a/5jdyg1q2/episode_rosbags/episode_3_2025-01-06-22-11-34.bag b/5jdyg1q2/episode_rosbags/episode_3_2025-01-06-22-11-34.bag new file mode 100644 index 0000000000000000000000000000000000000000..6fc50b2175a662a3590ca6f908e10a62c0c0cb9a --- /dev/null +++ b/5jdyg1q2/episode_rosbags/episode_3_2025-01-06-22-11-34.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f1ad55226767d6072259cd1b01b40296fa893bacce7e087817be0a58a674ad8 +size 1411902109 diff --git a/5w3hakcj/.hydra/config.yaml b/5w3hakcj/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab --- /dev/null +++ b/5w3hakcj/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1009_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/5w3hakcj/.hydra/hydra.yaml b/5w3hakcj/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..800ca20557842937ac4ca696bde020cd4db5cab4 --- /dev/null +++ b/5w3hakcj/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/5w3hakcj/.hydra/overrides.yaml b/5w3hakcj/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/5w3hakcj/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/5w3hakcj/episode_rosbags/aligned_depth_to_color_K.npy b/5w3hakcj/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/5w3hakcj/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/5w3hakcj/episode_rosbags/color_K.npy b/5w3hakcj/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/5w3hakcj/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/5w3hakcj/episode_rosbags/episode_0_2025-01-06-17-19-28.bag b/5w3hakcj/episode_rosbags/episode_0_2025-01-06-17-19-28.bag new file mode 100644 index 0000000000000000000000000000000000000000..5df095bffb554d56701eabe9e902037bf6609b7a --- /dev/null +++ b/5w3hakcj/episode_rosbags/episode_0_2025-01-06-17-19-28.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5430e59b62441f23f21ed0cd3e34ff533a7b72ed823731f35e28fc5b38fbe8d +size 1412408505 diff --git a/5w3hakcj/episode_rosbags/episode_1_2025-01-06-17-20-08.bag b/5w3hakcj/episode_rosbags/episode_1_2025-01-06-17-20-08.bag new file mode 100644 index 0000000000000000000000000000000000000000..dc6519eb0d561e230ad0eb793a30fa43deb8cf76 --- /dev/null +++ b/5w3hakcj/episode_rosbags/episode_1_2025-01-06-17-20-08.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bae4b8f16df48ff704dcef5879149bd5670c33a2297065e6945c5050885ba245 +size 1410316170 diff --git a/5w3hakcj/episode_rosbags/episode_2_2025-01-06-17-20-50.bag b/5w3hakcj/episode_rosbags/episode_2_2025-01-06-17-20-50.bag new file mode 100644 index 0000000000000000000000000000000000000000..96117c3233efe79778a910b99b728f0f97dddb19 --- /dev/null +++ b/5w3hakcj/episode_rosbags/episode_2_2025-01-06-17-20-50.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d393e51457a888d3c39ec79e3ec1c2970862d4e2b5970da3f34ed72a7a2535c8 +size 1413103128 diff --git a/5w3hakcj/episode_rosbags/episode_4_2025-01-06-17-22-22.bag b/5w3hakcj/episode_rosbags/episode_4_2025-01-06-17-22-22.bag new file mode 100644 index 0000000000000000000000000000000000000000..a2c252df4a2d435826fcb513f884671e25d42ec2 --- /dev/null +++ b/5w3hakcj/episode_rosbags/episode_4_2025-01-06-17-22-22.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d94b3c19dd225f7b0c8f11fb98d4dc3cc11358d4931346c6ff09bcdf63f94f7a +size 1412990439 diff --git a/5w3hakcj/eval_video/0_eval.mp4 b/5w3hakcj/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2ad3ec41ea4f7544b8970a1c3dfc26e194f70564 --- /dev/null +++ b/5w3hakcj/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:618aacdea903d28c41fe4c2f08ea34d3124dc62c6ca9ce20e16ca3ccbd4fddf1 +size 1135146 diff --git a/5w3hakcj/eval_video/1_eval.mp4 b/5w3hakcj/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..824cfba6877ee3bf0670e575e42d3efa82fcd6b9 --- /dev/null +++ b/5w3hakcj/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caf44f191ae0ae9f7deeb34d3ccf845fce7743d4bb573931f9176b3b7f273011 +size 1145064 diff --git a/5w3hakcj/eval_video/2_eval.mp4 b/5w3hakcj/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d48e907a05348451022bf2098289699e082764df --- /dev/null +++ b/5w3hakcj/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:493dcb44b239a781188a41ccaa2958375ee7d918c70f1342298d38e35cc0103e +size 1149577 diff --git a/5w3hakcj/eval_video/3_eval.mp4 b/5w3hakcj/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..96499ad8781dc00e33d0b8f0dba549d964782d6d --- /dev/null +++ b/5w3hakcj/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b633df55c1c901ea4269db186c946849d68c21b995e15e6ab13b223e5efdbac7 +size 1142856 diff --git a/5w3hakcj/eval_video/4_eval.mp4 b/5w3hakcj/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..22087e9846414cacf693193c3006cc6d18fc6d44 --- /dev/null +++ b/5w3hakcj/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4607322fe9bf46c96947e0676f6ccf9bcff1171b29587a1866c72ce380b8bf57 +size 1178538 diff --git a/5w3hakcj/tb/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 b/5w3hakcj/tb/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 new file mode 100644 index 0000000000000000000000000000000000000000..0e567a2250ef3208e64cec369dd68c1d78636e9f --- /dev/null +++ b/5w3hakcj/tb/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6b4c463a0c314bdf35309cfb8db610ce1bf7b231aa379086bd919728c059214 +size 6539 diff --git a/5w3hakcj/wandb/debug-internal.log b/5w3hakcj/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f28f00ba7f1a6c4ee4a25a6db436cd2b9d653a6a --- /dev/null +++ b/5w3hakcj/wandb/debug-internal.log @@ -0,0 +1,759 @@ +2025-01-06 17:18:49,535 INFO StreamThr :2088223 [internal.py:wandb_internal():85] W&B internal server running at pid: 2088223, started at: 2025-01-06 17:18:49.535087 +2025-01-06 17:18:49,536 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status +2025-01-06 17:18:49,547 INFO WriterThread:2088223 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/run-5w3hakcj.wandb +2025-01-06 17:18:49,548 DEBUG SenderThread:2088223 [sender.py:send():379] send: header +2025-01-06 17:18:49,558 DEBUG SenderThread:2088223 [sender.py:send():379] send: run +2025-01-06 17:18:49,769 INFO SenderThread:2088223 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files +2025-01-06 17:18:49,769 INFO SenderThread:2088223 [sender.py:_start_run_threads():1188] run started: 5w3hakcj with start time 1736201929.536184 +2025-01-06 17:18:49,788 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 17:18:49,788 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: check_version +2025-01-06 17:18:49,846 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 17:18:49,851 DEBUG HandlerThread:2088223 [system_info.py:__init__():26] System info init +2025-01-06 17:18:49,851 DEBUG HandlerThread:2088223 [system_info.py:__init__():41] System info init done +2025-01-06 17:18:49,851 INFO HandlerThread:2088223 [system_monitor.py:start():194] Starting system monitor +2025-01-06 17:18:49,851 INFO SystemMonitor:2088223 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 17:18:49,851 INFO HandlerThread:2088223 [system_monitor.py:probe():214] Collecting system info +2025-01-06 17:18:49,852 INFO SystemMonitor:2088223 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 17:18:49,852 INFO SystemMonitor:2088223 [interfaces.py:start():188] Started disk monitoring +2025-01-06 17:18:49,853 INFO SystemMonitor:2088223 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 17:18:49,854 INFO SystemMonitor:2088223 [interfaces.py:start():188] Started memory monitoring +2025-01-06 17:18:49,854 INFO SystemMonitor:2088223 [interfaces.py:start():188] Started network monitoring +2025-01-06 17:18:49,866 DEBUG HandlerThread:2088223 [system_info.py:probe():152] Probing system +2025-01-06 17:18:49,868 DEBUG HandlerThread:2088223 [system_info.py:_probe_git():137] Probing git +2025-01-06 17:18:49,872 DEBUG HandlerThread:2088223 [system_info.py:_probe_git():145] Probing git done +2025-01-06 17:18:49,872 DEBUG HandlerThread:2088223 [system_info.py:probe():200] Probing system done +2025-01-06 17:18:49,872 DEBUG HandlerThread:2088223 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:18:49.866525', 'startedAt': '2025-01-06T22:18:49.525800', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.59225, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.392, 'min': 800.0, 'max': 4700.0}, {'current': 4497.024, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 327.4358215332031}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 17:18:49,873 INFO HandlerThread:2088223 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 17:18:49,873 INFO HandlerThread:2088223 [system_monitor.py:probe():227] Publishing system info +2025-01-06 17:18:49,873 DEBUG HandlerThread:2088223 [system_info.py:_save_code():44] Saving code +2025-01-06 17:18:49,874 DEBUG HandlerThread:2088223 [system_info.py:_save_code():67] Saving code done +2025-01-06 17:18:49,874 DEBUG HandlerThread:2088223 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 17:18:49,936 DEBUG HandlerThread:2088223 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 17:18:49,936 INFO HandlerThread:2088223 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 17:18:49,939 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:18:49,939 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 17:18:49,939 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 17:18:49,940 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 17:18:50,165 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/gt8m6bqs-wandb-metadata.json +2025-01-06 17:18:50,185 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 17:18:50,186 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: python_packages +2025-01-06 17:18:50,188 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:18:50,188 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:18:50,199 INFO wandb-upload_1:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/n1buykb3-code/FISH/eval_robot.py +2025-01-06 17:18:50,229 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:50,229 INFO HandlerThread:2088223 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/tb" + save: true +} +control { + relay_id: "e26289fcd4a2434da0e8d8aba699564f" +} +_info { + stream_id: "5w3hakcj" +} + +2025-01-06 17:18:50,229 DEBUG HandlerThread:2088223 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 17:18:50,234 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/e2mxru82-diff.patch +2025-01-06 17:18:50,267 DEBUG SenderThread:2088223 [sender.py:send():379] send: telemetry +2025-01-06 17:18:50,280 DEBUG SenderThread:2088223 [sender.py:send():379] send: tbrecord +2025-01-06 17:18:50,281 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:18:50,281 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 with policy live +2025-01-06 17:18:50,455 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/3r60g6da-events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:18:50,771 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/diff.patch +2025-01-06 17:18:50,771 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/code/FISH/eval_robot.py +2025-01-06 17:18:50,771 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-metadata.json +2025-01-06 17:18:50,771 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/requirements.txt +2025-01-06 17:18:50,772 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:18:50,772 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:18:50,772 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/code/FISH +2025-01-06 17:18:50,772 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/code +2025-01-06 17:18:51,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:52,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:52,770 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:18:53,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:54,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:55,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:18:55,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:56,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:57,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:58,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:59,188 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:59,899 DEBUG SenderThread:2088223 [sender.py:send():379] send: config +2025-01-06 17:19:00,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:00,188 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:01,188 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:02,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:02,773 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:03,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:04,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:04,774 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:05,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:19:05,186 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:19:05,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:05,295 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:06,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:06,774 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:07,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:08,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:08,775 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:09,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:10,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:11,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:11,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:12,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:13,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:14,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:15,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:16,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:16,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:17,191 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:18,191 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:19,191 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:20,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:19:20,186 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:19:20,229 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:21,191 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:21,290 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:21,778 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/config.yaml +2025-01-06 17:19:22,191 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:23,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:24,191 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:25,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:26,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:26,521 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:26,780 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:27,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:28,233 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:29,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:30,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:30,781 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:31,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:32,188 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:32,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:33,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:34,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:34,782 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:35,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:19:35,186 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:19:35,229 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:36,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:37,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:37,291 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:38,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:39,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:40,203 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:41,203 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:42,203 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:43,188 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:43,203 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:44,203 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:45,203 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:46,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:47,206 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:48,188 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:48,206 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:49,206 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:49,855 DEBUG SystemMonitor:2088223 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 17:19:49,855 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:19:50,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:19:50,186 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:19:50,206 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:50,795 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:51,206 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:52,207 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:53,207 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:53,236 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:54,207 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:55,229 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:56,122 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:19:56,123 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 with policy now +2025-01-06 17:19:56,123 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:19:56,124 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:19:56,124 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:19:56,124 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:19:56,207 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:56,329 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/9g4zrg69-media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 +2025-01-06 17:19:56,797 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:56,797 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:19:56,797 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 +2025-01-06 17:19:56,797 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media +2025-01-06 17:19:56,797 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos +2025-01-06 17:19:56,797 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval +2025-01-06 17:19:57,059 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/0jamwfhf-media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 +2025-01-06 17:19:57,207 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:58,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:58,797 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:59,131 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:59,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:00,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:00,798 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:01,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:02,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:03,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:04,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:04,442 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:04,799 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:04,800 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:20:05,049 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/5e0faf8q-events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:20:05,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:20:05,186 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:20:05,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:06,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:06,800 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:07,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:08,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:09,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:10,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:10,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:10,802 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:11,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:12,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:13,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:14,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:15,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:15,210 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:16,210 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:17,210 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:18,210 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:19,210 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:19,856 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:20:20,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:20:20,187 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:20:20,233 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:20,245 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:21,210 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:22,210 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:23,211 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:24,211 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:25,211 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:26,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:26,211 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:27,211 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:28,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:29,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:30,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:31,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:31,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:32,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:33,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:34,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:35,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:20:35,187 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:20:35,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:36,204 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:20:36,205 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 with policy now +2025-01-06 17:20:36,205 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:36,205 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:20:36,206 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:20:36,206 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:20:36,208 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:20:36,215 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:36,412 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/4r8at4h6-media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 +2025-01-06 17:20:36,810 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:20:36,810 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:36,810 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 +2025-01-06 17:20:36,810 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval +2025-01-06 17:20:37,068 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/0w05fpom-media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 +2025-01-06 17:20:37,216 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:38,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:38,810 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:39,227 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:40,216 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:40,811 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:41,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:42,194 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:42,217 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:43,216 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:44,216 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:44,812 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:45,216 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:46,217 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:46,813 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:20:47,216 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:47,506 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:47,641 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:20:47,641 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:20:47,641 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:20:47,813 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:20:48,217 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:48,814 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:49,217 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:49,857 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:20:50,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:20:50,188 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:20:50,229 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:51,218 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:52,218 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:52,815 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:53,218 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:53,484 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:54,218 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:55,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:56,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:57,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:58,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:58,817 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:58,861 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:59,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:00,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:01,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:02,220 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:03,220 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:04,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:04,220 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:05,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:21:05,187 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:21:05,229 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:06,220 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:07,220 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:08,220 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:09,220 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:10,077 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:10,221 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:10,821 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:21:11,221 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:12,221 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:13,221 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:14,221 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:15,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:15,221 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:16,221 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:17,233 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:18,223 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:18,413 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:21:18,413 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:21:18,414 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 with policy now +2025-01-06 17:21:18,414 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:21:18,414 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:21:18,415 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:21:18,631 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/6xakmz00-media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 +2025-01-06 17:21:18,824 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:21:18,824 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:21:18,824 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 +2025-01-06 17:21:18,824 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval +2025-01-06 17:21:19,078 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/fcewp0dk-media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 +2025-01-06 17:21:19,226 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:19,857 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:21:20,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:21:20,193 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:21:20,237 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:20,270 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:20,825 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:21:21,234 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:22,229 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:23,235 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:24,233 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:24,826 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:21:25,232 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:26,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:26,234 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:26,827 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:21:27,237 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:28,239 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:29,242 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:30,242 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:31,199 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:31,242 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:32,246 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:33,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:34,248 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:34,829 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:21:35,070 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/fjcxvmgt-events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:21:35,191 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:21:35,191 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:21:35,252 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:35,479 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:21:35,479 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:21:35,479 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:21:35,830 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:21:36,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:36,785 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:36,830 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:21:37,255 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:38,254 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:39,249 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:40,249 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:40,831 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:21:41,249 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:42,194 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:42,249 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:43,249 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:44,249 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:45,249 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:46,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:47,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:48,194 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:48,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:49,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:49,858 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:21:50,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:21:50,190 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:21:50,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:51,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:52,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:53,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:53,325 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:54,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:55,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:56,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:57,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:58,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:59,194 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:59,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:00,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:01,252 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:02,252 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:03,252 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:04,195 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:04,252 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:05,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:22:05,190 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:22:05,252 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:06,255 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:06,840 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:07,141 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:22:07,141 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 with policy now +2025-01-06 17:22:07,160 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:22:07,160 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:22:07,160 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:22:07,161 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:22:07,257 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:07,346 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/1lcx2tqf-media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 +2025-01-06 17:22:07,841 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:22:07,841 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 +2025-01-06 17:22:07,841 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval +2025-01-06 17:22:08,101 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/0pavcjfu-media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 +2025-01-06 17:22:08,261 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:08,841 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:09,262 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:09,841 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:10,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:10,261 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:11,259 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:12,267 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:12,842 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:13,264 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:14,262 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:14,843 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:15,197 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:15,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:16,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:17,267 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:18,263 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:19,261 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:19,530 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:22:19,530 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:22:19,530 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:22:19,845 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:22:19,845 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:22:19,858 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:22:20,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:22:20,193 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:22:20,265 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:20,341 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:20,845 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:21,267 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:22,265 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:22,846 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:23,268 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:24,265 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:24,847 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:25,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:26,196 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:26,265 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:26,847 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:27,265 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:28,265 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:29,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:30,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:31,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:32,196 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:32,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:33,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:34,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:35,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:22:35,190 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:22:35,267 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:36,267 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:37,240 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:37,267 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:38,267 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:39,268 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:40,268 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:41,268 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:42,268 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:43,196 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:43,268 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:44,268 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:45,269 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:46,269 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:47,269 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:48,197 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:48,269 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:49,269 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:49,859 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:22:50,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:22:50,190 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:22:50,269 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:51,273 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:51,912 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:22:51,913 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 with policy now +2025-01-06 17:22:51,917 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:22:51,918 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:22:51,918 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:22:51,918 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:22:52,130 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/9k26rq02-media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 +2025-01-06 17:22:52,272 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:52,856 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:22:52,857 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:52,857 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 +2025-01-06 17:22:52,857 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval +2025-01-06 17:22:53,129 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/x4lwqnba-media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 +2025-01-06 17:22:53,281 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:53,936 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:54,280 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:55,279 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:56,280 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:56,858 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:57,286 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:58,283 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:59,290 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:59,377 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:23:00,289 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:00,859 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:23:01,288 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:02,294 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:03,299 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:04,293 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:05,202 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:23:05,202 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:23:05,276 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:23:05,291 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:05,861 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:23:06,112 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/epgz5fhh-events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:23:06,280 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:23:06,280 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:23:06,294 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:06,460 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:23:06,725 INFO wandb-upload_0:2088223 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpt9ubxmqi +2025-01-06 17:23:06,809 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:23:06,822 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:23:06,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:23:06,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:23:06,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 17:23:06,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-06 17:23:06,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media +2025-01-06 17:23:06,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table +2025-01-06 17:23:06,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval +2025-01-06 17:23:07,102 INFO SenderThread:2088223 [sender.py:send_request_log_artifact():1525] logged artifact run-5w3hakcj-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTM0Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMDU5Mw==', 'latestArtifact': None}} +2025-01-06 17:23:07,102 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:23:07,102 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-06 17:23:07,104 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:23:07,104 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,104 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,104 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:23:07,105 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,108 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,108 DEBUG SenderThread:2088223 [sender.py:send():379] send: telemetry +2025-01-06 17:23:07,109 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,109 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,109 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,109 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,109 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,110 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,110 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,110 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,110 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,111 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,111 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,111 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,111 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,112 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,112 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,112 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,112 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,113 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,113 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,113 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,113 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,113 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,114 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:23:07,267 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/b8bdhkrj-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 17:23:07,290 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:07,324 INFO wandb-upload_1:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/kirrr60p-media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-06 17:23:07,484 INFO wandb-upload_0:2088223 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_goqsqj8 +2025-01-06 17:23:07,514 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/xs3f6cik-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 17:23:07,845 INFO SenderThread:2088223 [sender.py:send_request_log_artifact():1525] logged artifact run-5w3hakcj-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTM1OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMDU5NQ==', 'latestArtifact': None}} +2025-01-06 17:23:07,845 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:23:07,845 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json with policy now +2025-01-06 17:23:07,846 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:23:07,846 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,846 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:23:08,084 INFO wandb-upload_1:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/khasfmu7-media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-06 17:23:08,290 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:08,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:23:09,290 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:10,290 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:10,459 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:23:10,566 DEBUG SenderThread:2088223 [sender.py:send():379] send: exit +2025-01-06 17:23:10,567 INFO SenderThread:2088223 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 17:23:10,567 INFO SenderThread:2088223 [sender.py:send_exit():588] handling runtime: 260 +2025-01-06 17:23:10,568 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:10,568 INFO SenderThread:2088223 [sender.py:send_exit():594] send defer +2025-01-06 17:23:10,568 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:10,568 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 17:23:10,568 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:10,568 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 17:23:10,568 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 1 +2025-01-06 17:23:10,568 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:10,569 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 17:23:10,569 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:10,569 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 17:23:10,569 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 2 +2025-01-06 17:23:10,569 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:10,569 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 17:23:10,569 INFO HandlerThread:2088223 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 17:23:10,569 DEBUG SystemMonitor:2088223 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 17:23:10,569 DEBUG SystemMonitor:2088223 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 17:23:10,570 INFO HandlerThread:2088223 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 17:23:10,570 INFO HandlerThread:2088223 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 17:23:10,623 INFO HandlerThread:2088223 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 17:23:10,623 INFO HandlerThread:2088223 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 17:23:10,624 INFO HandlerThread:2088223 [interfaces.py:finish():200] Joined network monitor +2025-01-06 17:23:10,624 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:10,624 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 17:23:10,624 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 3 +2025-01-06 17:23:10,624 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:23:10,624 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:10,624 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 17:23:10,625 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:10,625 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 17:23:10,625 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 4 +2025-01-06 17:23:10,625 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:10,625 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 17:23:10,863 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/config.yaml +2025-01-06 17:23:10,863 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:23:12,864 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:23:17,592 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:23:17,592 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:17,592 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:23:17,593 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 17:23:17,593 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 17:23:17,594 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 5 +2025-01-06 17:23:17,594 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:23:17,594 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:23:17,594 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:17,594 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 17:23:17,594 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:17,595 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:17,595 DEBUG SenderThread:2088223 [sender.py:send():379] send: summary +2025-01-06 17:23:17,596 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:17,596 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:17,596 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 17:23:17,596 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 6 +2025-01-06 17:23:17,596 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:17,596 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 17:23:17,596 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:17,596 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 17:23:17,596 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 7 +2025-01-06 17:23:17,596 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:23:17,597 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:17,597 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 17:23:17,597 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:17,597 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 17:23:17,866 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:23:18,503 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:23:19,077 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 8 +2025-01-06 17:23:19,077 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:23:19,077 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:19,078 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 17:23:19,078 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:19,078 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 17:23:19,078 INFO SenderThread:2088223 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 17:23:19,080 INFO SenderThread:2088223 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 17:23:19,082 INFO SenderThread:2088223 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 17:23:19,090 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 9 +2025-01-06 17:23:19,091 DEBUG SenderThread:2088223 [sender.py:send():379] send: artifact +2025-01-06 17:23:19,093 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:19,093 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 17:23:19,450 INFO wandb-upload_1:2088223 [upload_job.py:push():85] Skipped uploading /tmp/tmpxdqw67u_/wandb-job.json +2025-01-06 17:23:19,450 INFO wandb-upload_0:2088223 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdz6bjyvd +2025-01-06 17:23:19,503 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:23:19,564 INFO wandb-upload_2:2088223 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp49htejn7 +2025-01-06 17:23:19,867 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:23:20,102 INFO SenderThread:2088223 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'versionIndex': 422}}} +2025-01-06 17:23:20,102 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:20,102 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 17:23:20,102 INFO SenderThread:2088223 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 17:23:20,868 INFO SenderThread:2088223 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files +2025-01-06 17:23:20,868 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/config.yaml config.yaml +2025-01-06 17:23:20,869 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json wandb-summary.json +2025-01-06 17:23:20,869 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:23:20,871 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/requirements.txt requirements.txt +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-metadata.json wandb-metadata.json +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log output.log +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/diff.patch diff.patch +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 +2025-01-06 17:23:20,874 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 +2025-01-06 17:23:20,874 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-06 17:23:20,875 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 17:23:20,875 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 10 +2025-01-06 17:23:20,875 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:23:20,876 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:20,880 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 17:23:20,880 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:20,880 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 17:23:20,880 INFO SenderThread:2088223 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:23:21,039 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:23:21,056 INFO wandb-upload_1:2088223 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/config.yaml +2025-01-06 17:23:21,102 INFO wandb-upload_4:2088223 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:23:21,120 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/requirements.txt +2025-01-06 17:23:21,143 INFO wandb-upload_3:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/aw4645rl-events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:23:21,343 INFO Thread-11 (_thread_body):2088223 [sender.py:transition_state():614] send defer: 11 +2025-01-06 17:23:21,343 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:21,343 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 17:23:21,343 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:21,343 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 17:23:21,344 INFO SenderThread:2088223 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 17:23:21,345 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 12 +2025-01-06 17:23:21,345 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:21,345 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 17:23:21,345 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:21,345 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 17:23:21,345 INFO SenderThread:2088223 [file_stream.py:finish():601] file stream finish called +2025-01-06 17:23:21,406 INFO SenderThread:2088223 [file_stream.py:finish():605] file stream finish is done +2025-01-06 17:23:21,407 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 13 +2025-01-06 17:23:21,407 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:21,407 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 17:23:21,407 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:21,407 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 17:23:21,407 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 14 +2025-01-06 17:23:21,407 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:21,407 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 17:23:21,407 DEBUG SenderThread:2088223 [sender.py:send():379] send: final +2025-01-06 17:23:21,408 DEBUG SenderThread:2088223 [sender.py:send():379] send: footer +2025-01-06 17:23:21,408 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:21,408 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 17:23:21,408 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:23:21,408 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:23:21,408 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 17:23:21,409 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:23:21,409 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: server_info +2025-01-06 17:23:21,410 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 17:23:21,411 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 17:23:21,411 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:21,465 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:23:21,465 INFO MainThread:2088223 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 17:23:21,465 INFO MainThread:2088223 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 17:23:21,466 INFO MainThread:2088223 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 17:23:21,466 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 17:23:21,466 INFO HandlerThread:2088223 [handler.py:finish():882] shutting down handler +2025-01-06 17:23:22,409 INFO WriterThread:2088223 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/run-5w3hakcj.wandb +2025-01-06 17:23:22,465 INFO SenderThread:2088223 [sender.py:finish():1615] shutting down sender +2025-01-06 17:23:22,465 INFO SenderThread:2088223 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:23:22,465 INFO SenderThread:2088223 [file_pusher.py:join():175] waiting for file pusher diff --git a/5w3hakcj/wandb/debug.log b/5w3hakcj/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c25d9af0d9313f53f74b7a7b52b29f79c858cbb3 --- /dev/null +++ b/5w3hakcj/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 17:18:49,528 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 17:18:49,528 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Configure stats pid to 2088037 +2025-01-06 17:18:49,528 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/settings +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/logs/debug.log +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/logs/debug-internal.log +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_init.py:init():569] calling init triggers +2025-01-06 17:18:49,530 INFO MainThread:2088037 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj'} +2025-01-06 17:18:49,530 INFO MainThread:2088037 [wandb_init.py:init():619] starting backend +2025-01-06 17:18:49,530 INFO MainThread:2088037 [wandb_init.py:init():623] setting up manager +2025-01-06 17:18:49,534 INFO MainThread:2088037 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 17:18:49,535 INFO MainThread:2088037 [wandb_init.py:init():631] backend started and connected +2025-01-06 17:18:49,547 INFO MainThread:2088037 [wandb_init.py:init():720] updated telemetry +2025-01-06 17:18:49,556 INFO MainThread:2088037 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 17:18:49,787 INFO MainThread:2088037 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 17:18:49,841 INFO MainThread:2088037 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 17:18:49,841 INFO MainThread:2088037 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 17:18:50,186 INFO MainThread:2088037 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 17:18:50,186 INFO MainThread:2088037 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 17:18:50,186 INFO MainThread:2088037 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 17:18:50,187 INFO MainThread:2088037 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 17:18:50,188 INFO MainThread:2088037 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 17:18:50,189 INFO MainThread:2088037 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/tb, True +2025-01-06 17:18:59,898 INFO MainThread:2088037 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 17:23:22,468 WARNING MsgRouterThr:2088037 [router.py:message_loop():77] message_loop has been closed diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/code/FISH/eval_robot.py b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/config.yaml b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d7df50ad066282f773f2f2f4257e25a18c266ff --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1009_1' +wandb_notes: + desc: null + value: 1009_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736201929 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/diff.patch b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..43dd1a505c5d25a547a09b1d8e6dc31eb2a6a985 --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..101f9bd 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0cf4c88c517af93d38ad4b5d570fa02efc2afe50 --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[5, 5, 1.0, 0.9786591528888874, 1, 0.5107108652570282, 1, 0.5613806098576808, 1, 1, 1]]} \ No newline at end of file diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2ad3ec41ea4f7544b8970a1c3dfc26e194f70564 --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:618aacdea903d28c41fe4c2f08ea34d3124dc62c6ca9ce20e16ca3ccbd4fddf1 +size 1135146 diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..824cfba6877ee3bf0670e575e42d3efa82fcd6b9 --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caf44f191ae0ae9f7deeb34d3ccf845fce7743d4bb573931f9176b3b7f273011 +size 1145064 diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d48e907a05348451022bf2098289699e082764df --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:493dcb44b239a781188a41ccaa2958375ee7d918c70f1342298d38e35cc0103e +size 1149577 diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..96499ad8781dc00e33d0b8f0dba549d964782d6d --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b633df55c1c901ea4269db186c946849d68c21b995e15e6ab13b223e5efdbac7 +size 1142856 diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..22087e9846414cacf693193c3006cc6d18fc6d44 --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4607322fe9bf46c96947e0676f6ccf9bcff1171b29587a1866c72ce380b8bf57 +size 1178538 diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..cbecb10d20096ed36a552fc2d3f192d71eda65c5 --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log @@ -0,0 +1,269 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736201939.899109]: resetting environment +[INFO] [1736201939.902626]: cleared current plan +[INFO] [1736201939.903655]: moving to home +[INFO] [1736201940.905267]: reached home +[INFO] [1736201940.906559]: reset action history +[INFO] [1736201943.667308]: environment reset +Starting episode 0 +[INFO] [1736201943.669471]: resetting environment +[INFO] [1736201943.672806]: cleared current plan +[INFO] [1736201943.674971]: moving to home +[INFO] [1736201944.677265]: reached home +[INFO] [1736201944.678857]: reset action history +[INFO] [1736201947.387019]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736201964.474344]: resetting environment +[INFO] [1736201964.477820]: cleared current plan +[INFO] [1736201964.478648]: moving to home +[INFO] [1736201965.480972]: reached home +[INFO] [1736201965.482215]: reset action history +[INFO] [1736201968.190387]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/episode_rosbags/episode_0_2025-01-06-17-19-28.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2088037.log +started roslaunch server http://158.130.50.37:42621/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2088328] +process[print_text-2]: started with pid [2088352] +process[rosbag_record-3]: started with pid [2088353] +started bagging! +For topic gripper_width: timestamp difference is 106482052 for nearest: 1736201972906801008 - target: 1736201972800318956 at idx 67 +For topic gripper_width: timestamp difference is 119910348 for nearest: 1736201989440130342 - target: 1736201989320219994 at idx 249 +For topic gripper_width: timestamp difference is -113229405 for nearest: 1736201989440130342 - target: 1736201989553359747 at idx 249 +[INFO] [1736201995.188644]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1736201995.916634]: Stored episode 1. +[INFO] [1736201995.916943]: Saving video... +[INFO] [1736201996.112325]: Video saved! +Starting episode 1 +[INFO] [1736201996.124786]: resetting environment +[INFO] [1736201996.130530]: cleared current plan +[INFO] [1736201996.130831]: moving to home +[INFO] [1736201999.734025]: reached home +[INFO] [1736201999.735584]: reset action history +[INFO] [1736202002.446393]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736202004.445467]: resetting environment +[INFO] [1736202004.447815]: cleared current plan +[INFO] [1736202004.448090]: moving to home +[INFO] [1736202005.449255]: reached home +[INFO] [1736202005.449898]: reset action history +[INFO] [1736202008.159321]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/episode_rosbags/episode_1_2025-01-06-17-20-08.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2088037.log +started roslaunch server http://158.130.50.37:41389/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2088463] +process[print_text-5]: started with pid [2088464] +process[rosbag_record-6]: started with pid [2088465] +started bagging! +[INFO] [1736202035.129508]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736202035.875483]: Stored episode 2. +[INFO] [1736202035.877537]: Saving video... +[INFO] [1736202036.187452]: Video saved! +Starting episode 2 +[INFO] [1736202036.207186]: resetting environment +[INFO] [1736202036.221199]: cleared current plan +[INFO] [1736202036.223309]: moving to home +[INFO] [1736202039.832341]: reached home +[INFO] [1736202039.834557]: reset action history +[INFO] [1736202042.613625]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736202046.484264]: resetting environment +[INFO] [1736202046.492332]: cleared current plan +[INFO] [1736202046.493507]: moving to home +[INFO] [1736202047.496458]: reached home +[INFO] [1736202047.498836]: reset action history +[INFO] [1736202050.208701]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/episode_rosbags/episode_2_2025-01-06-17-20-50.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2088037.log +started roslaunch server http://158.130.50.37:42407/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2088597] +process[print_text-8]: started with pid [2088621] +process[rosbag_record-9]: started with pid [2088622] +started bagging! +For topic gripper_width: timestamp difference is 107659579 for nearest: 1736202056806826592 - target: 1736202056699167013 at idx 85 +For topic gripper_width: timestamp difference is -115698920 for nearest: 1736202068840156688 - target: 1736202068955855608 at idx 240 +[INFO] [1736202077.190341]: Storing episode... +[print_text-8] killing on exit +[pub_text-7] killing on exit +[rosbag_record-9] killing on exit +[INFO] [1736202077.975102]: Stored episode 3. +[INFO] [1736202077.976574]: Saving video... +[INFO] [1736202078.370718]: Video saved! +Starting episode 3 +[INFO] [1736202078.407552]: resetting environment +[INFO] [1736202078.418788]: cleared current plan +[INFO] [1736202078.420940]: moving to home +[INFO] [1736202082.036448]: reached home +[INFO] [1736202082.039373]: reset action history +[INFO] [1736202084.812415]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736202094.712060]: resetting environment +[INFO] [1736202094.730237]: cleared current plan +[INFO] [1736202094.731722]: moving to home +[INFO] [1736202095.736463]: reached home +[INFO] [1736202095.740465]: reset action history +[INFO] [1736202098.667444]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/episode_rosbags/episode_3_2025-01-06-17-21-38.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2088037.log +started roslaunch server http://158.130.50.37:35845/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2088735] +process[print_text-11]: started with pid [2088736] +process[rosbag_record-12]: started with pid [2088760] +started bagging! +[INFO] [1736202125.651038]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1736202126.412196]: Stored episode 4. +[INFO] [1736202126.418123]: Saving video... +[INFO] [1736202127.094316]: Video saved! +Starting episode 4 +[INFO] [1736202127.170446]: resetting environment +[INFO] [1736202127.187445]: cleared current plan +[INFO] [1736202127.191066]: moving to home +[WARN] [1736202127.005476]: Plan exhausted +[WARN] [1736202127.048152]: Plan exhausted +[WARN] [1736202127.084314]: Plan exhausted +[WARN] [1736202127.125456]: Plan exhausted +[WARN] [1736202127.164531]: Plan exhausted +[INFO] [1736202130.798691]: reached home +[INFO] [1736202130.805386]: reset action history +[INFO] [1736202133.588274]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736202138.978118]: resetting environment +[INFO] [1736202138.988353]: cleared current plan +[INFO] [1736202138.988564]: moving to home +[INFO] [1736202139.990609]: reached home +[INFO] [1736202139.993519]: reset action history +[INFO] [1736202142.773356]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/episode_rosbags/episode_4_2025-01-06-17-22-22.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2088037.log +started roslaunch server http://158.130.50.37:36521/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2088894] +process[print_text-14]: started with pid [2088895] +process[rosbag_record-15]: started with pid [2088896] +started bagging! +[INFO] [1736202170.845202]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1736202171.634152]: Stored episode 5. +[INFO] [1736202171.637529]: Saving video... +[INFO] [1736202171.865004]: Video saved! +[INFO] [1736202171.920091]: resetting environment +[INFO] [1736202171.934999]: cleared current plan +[INFO] [1736202171.935306]: moving to home +[INFO] [1736202175.539706]: reached home +[INFO] [1736202175.545442]: reset action history +[INFO] [1736202178.326636]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/requirements.txt b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-metadata.json b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..04e4110f2a517fc339d05bdefa6d57fe873ddc13 --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-06T22:18:49.866525", + "startedAt": "2025-01-06T22:18:49.525800", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.59225, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.392, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.024, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 327.4358215332031 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..54310768049b71e5ddc002a0b472671aabd33f48 --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "618aacdea903d28c41fe4c2f08ea34d3124dc62c6ca9ce20e16ca3ccbd4fddf1", "size": 1135146, "path": "media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4"}, "global_step": 664, "_timestamp": 1736202186.7315466, "_runtime": 257.1953625679016, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "caf44f191ae0ae9f7deeb34d3ccf845fce7743d4bb573931f9176b3b7f273011", "size": 1145064, "path": "media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4"}, "eval/num_success": 5.0, "episode": 5.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "493dcb44b239a781188a41ccaa2958375ee7d918c70f1342298d38e35cc0103e", "size": 1149577, "path": "media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "b633df55c1c901ea4269db186c946849d68c21b995e15e6ab13b223e5efdbac7", "size": 1142856, "path": "media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "4607322fe9bf46c96947e0676f6ccf9bcff1171b29587a1866c72ce380b8bf57", "size": 1178538, "path": "media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://mjc1ai219p7zqg6l1l9mt0ouhvjgslsyqx51tnzllwsvr2fyxu9t3fgaje8tu78k40mikuqk5d6ogds9ygb5pajur5xqcpkfgfyxw87l82c6kxqm09jkdw2zz7qq0bqn/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://49zhtwi1ecxbvjf9t58qoeqrl10ctn6pgcjcmnitqinwmfinllb7mr7wye6z7qk5e6m242q1ia9jjp16im9lqrzt13tb0ze3jp8es12iu1lugztiodi7p26856zpuygz:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 5, "total_num_episodes": 5, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9786591528888874, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.5107108652570282, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.5613806098576808, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "e3405f8336f17a4a59dd9de56afebbdceab20a8c5c8ce9c3ea21635b9f54eaac", "size": 418, "artifact_path": "wandb-client-artifact://gsf7evnjo8s0g0sobfoxbgryi3gz0rsr7pt50ae389y74c47ng03ryf5lxbvprq5o6cysc23ehz73iew9rs1zwq7aob4g3a9r2ree91zdop5s0t1pxghvd3w6u8g1vn0/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://klmhtemqvx21la4ji39bnptdo8nwphmfcco5b71vf6ac0v4pgjlu0b4dc4f8hhvdoe7ej2pspcaf8yrdqdqu605dr57gfs10q8dlioy6ooh81qzi1obzw1no8bt6yre7:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 260}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9786591529846191, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.5107108354568481, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.5613806247711182, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/logs/debug-internal.log b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f28f00ba7f1a6c4ee4a25a6db436cd2b9d653a6a --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/logs/debug-internal.log @@ -0,0 +1,759 @@ +2025-01-06 17:18:49,535 INFO StreamThr :2088223 [internal.py:wandb_internal():85] W&B internal server running at pid: 2088223, started at: 2025-01-06 17:18:49.535087 +2025-01-06 17:18:49,536 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status +2025-01-06 17:18:49,547 INFO WriterThread:2088223 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/run-5w3hakcj.wandb +2025-01-06 17:18:49,548 DEBUG SenderThread:2088223 [sender.py:send():379] send: header +2025-01-06 17:18:49,558 DEBUG SenderThread:2088223 [sender.py:send():379] send: run +2025-01-06 17:18:49,769 INFO SenderThread:2088223 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files +2025-01-06 17:18:49,769 INFO SenderThread:2088223 [sender.py:_start_run_threads():1188] run started: 5w3hakcj with start time 1736201929.536184 +2025-01-06 17:18:49,788 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 17:18:49,788 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: check_version +2025-01-06 17:18:49,846 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 17:18:49,851 DEBUG HandlerThread:2088223 [system_info.py:__init__():26] System info init +2025-01-06 17:18:49,851 DEBUG HandlerThread:2088223 [system_info.py:__init__():41] System info init done +2025-01-06 17:18:49,851 INFO HandlerThread:2088223 [system_monitor.py:start():194] Starting system monitor +2025-01-06 17:18:49,851 INFO SystemMonitor:2088223 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 17:18:49,851 INFO HandlerThread:2088223 [system_monitor.py:probe():214] Collecting system info +2025-01-06 17:18:49,852 INFO SystemMonitor:2088223 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 17:18:49,852 INFO SystemMonitor:2088223 [interfaces.py:start():188] Started disk monitoring +2025-01-06 17:18:49,853 INFO SystemMonitor:2088223 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 17:18:49,854 INFO SystemMonitor:2088223 [interfaces.py:start():188] Started memory monitoring +2025-01-06 17:18:49,854 INFO SystemMonitor:2088223 [interfaces.py:start():188] Started network monitoring +2025-01-06 17:18:49,866 DEBUG HandlerThread:2088223 [system_info.py:probe():152] Probing system +2025-01-06 17:18:49,868 DEBUG HandlerThread:2088223 [system_info.py:_probe_git():137] Probing git +2025-01-06 17:18:49,872 DEBUG HandlerThread:2088223 [system_info.py:_probe_git():145] Probing git done +2025-01-06 17:18:49,872 DEBUG HandlerThread:2088223 [system_info.py:probe():200] Probing system done +2025-01-06 17:18:49,872 DEBUG HandlerThread:2088223 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:18:49.866525', 'startedAt': '2025-01-06T22:18:49.525800', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.59225, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.392, 'min': 800.0, 'max': 4700.0}, {'current': 4497.024, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 327.4358215332031}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 17:18:49,873 INFO HandlerThread:2088223 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 17:18:49,873 INFO HandlerThread:2088223 [system_monitor.py:probe():227] Publishing system info +2025-01-06 17:18:49,873 DEBUG HandlerThread:2088223 [system_info.py:_save_code():44] Saving code +2025-01-06 17:18:49,874 DEBUG HandlerThread:2088223 [system_info.py:_save_code():67] Saving code done +2025-01-06 17:18:49,874 DEBUG HandlerThread:2088223 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 17:18:49,936 DEBUG HandlerThread:2088223 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 17:18:49,936 INFO HandlerThread:2088223 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 17:18:49,939 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:18:49,939 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 17:18:49,939 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 17:18:49,940 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 17:18:50,165 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/gt8m6bqs-wandb-metadata.json +2025-01-06 17:18:50,185 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 17:18:50,186 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: python_packages +2025-01-06 17:18:50,188 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:18:50,188 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:18:50,199 INFO wandb-upload_1:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/n1buykb3-code/FISH/eval_robot.py +2025-01-06 17:18:50,229 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:50,229 INFO HandlerThread:2088223 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/tb" + save: true +} +control { + relay_id: "e26289fcd4a2434da0e8d8aba699564f" +} +_info { + stream_id: "5w3hakcj" +} + +2025-01-06 17:18:50,229 DEBUG HandlerThread:2088223 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 17:18:50,234 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/e2mxru82-diff.patch +2025-01-06 17:18:50,267 DEBUG SenderThread:2088223 [sender.py:send():379] send: telemetry +2025-01-06 17:18:50,280 DEBUG SenderThread:2088223 [sender.py:send():379] send: tbrecord +2025-01-06 17:18:50,281 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:18:50,281 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 with policy live +2025-01-06 17:18:50,455 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/3r60g6da-events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:18:50,771 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/diff.patch +2025-01-06 17:18:50,771 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/code/FISH/eval_robot.py +2025-01-06 17:18:50,771 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-metadata.json +2025-01-06 17:18:50,771 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/requirements.txt +2025-01-06 17:18:50,772 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:18:50,772 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:18:50,772 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/code/FISH +2025-01-06 17:18:50,772 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/code +2025-01-06 17:18:51,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:52,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:52,770 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:18:53,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:54,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:55,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:18:55,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:56,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:57,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:58,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:59,188 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:59,899 DEBUG SenderThread:2088223 [sender.py:send():379] send: config +2025-01-06 17:19:00,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:00,188 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:01,188 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:02,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:02,773 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:03,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:04,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:04,774 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:05,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:19:05,186 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:19:05,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:05,295 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:06,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:06,774 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:07,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:08,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:08,775 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:09,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:10,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:11,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:11,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:12,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:13,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:14,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:15,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:16,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:16,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:17,191 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:18,191 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:19,191 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:20,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:19:20,186 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:19:20,229 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:21,191 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:21,290 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:21,778 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/config.yaml +2025-01-06 17:19:22,191 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:23,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:24,191 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:25,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:26,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:26,521 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:26,780 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:27,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:28,233 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:29,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:30,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:30,781 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:31,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:32,188 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:32,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:33,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:34,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:34,782 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:35,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:19:35,186 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:19:35,229 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:36,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:37,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:37,291 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:38,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:39,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:40,203 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:41,203 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:42,203 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:43,188 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:43,203 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:44,203 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:45,203 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:46,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:47,206 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:48,188 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:48,206 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:49,206 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:49,855 DEBUG SystemMonitor:2088223 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 17:19:49,855 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:19:50,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:19:50,186 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:19:50,206 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:50,795 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:51,206 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:52,207 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:53,207 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:53,236 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:54,207 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:55,229 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:56,122 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:19:56,123 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 with policy now +2025-01-06 17:19:56,123 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:19:56,124 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:19:56,124 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:19:56,124 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:19:56,207 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:56,329 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/9g4zrg69-media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 +2025-01-06 17:19:56,797 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:56,797 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:19:56,797 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 +2025-01-06 17:19:56,797 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media +2025-01-06 17:19:56,797 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos +2025-01-06 17:19:56,797 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval +2025-01-06 17:19:57,059 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/0jamwfhf-media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 +2025-01-06 17:19:57,207 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:58,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:19:58,797 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:19:59,131 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:19:59,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:00,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:00,798 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:01,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:02,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:03,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:04,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:04,442 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:04,799 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:04,800 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:20:05,049 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/5e0faf8q-events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:20:05,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:20:05,186 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:20:05,208 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:06,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:06,800 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:07,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:08,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:09,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:10,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:10,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:10,802 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:11,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:12,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:13,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:14,209 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:15,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:15,210 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:16,210 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:17,210 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:18,210 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:19,210 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:19,856 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:20:20,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:20:20,187 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:20:20,233 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:20,245 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:21,210 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:22,210 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:23,211 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:24,211 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:25,211 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:26,189 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:26,211 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:27,211 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:28,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:29,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:30,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:31,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:31,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:32,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:33,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:34,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:35,186 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:20:35,187 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:20:35,212 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:36,204 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:20:36,205 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 with policy now +2025-01-06 17:20:36,205 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:36,205 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:20:36,206 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:20:36,206 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:20:36,208 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:20:36,215 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:36,412 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/4r8at4h6-media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 +2025-01-06 17:20:36,810 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:20:36,810 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:36,810 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 +2025-01-06 17:20:36,810 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval +2025-01-06 17:20:37,068 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/0w05fpom-media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 +2025-01-06 17:20:37,216 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:38,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:38,810 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:39,227 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:40,216 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:40,811 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:41,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:42,194 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:42,217 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:43,216 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:44,216 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:44,812 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:45,216 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:46,217 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:46,813 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:20:47,216 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:47,506 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:47,641 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:20:47,641 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:20:47,641 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:20:47,813 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:20:48,217 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:48,814 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:49,217 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:49,857 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:20:50,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:20:50,188 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:20:50,229 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:51,218 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:52,218 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:52,815 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:53,218 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:53,484 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:54,218 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:55,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:56,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:57,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:58,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:20:58,817 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:20:58,861 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:20:59,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:00,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:01,219 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:02,220 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:03,220 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:04,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:04,220 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:05,187 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:21:05,187 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:21:05,229 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:06,220 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:07,220 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:08,220 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:09,220 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:10,077 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:10,221 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:10,821 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:21:11,221 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:12,221 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:13,221 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:14,221 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:15,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:15,221 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:16,221 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:17,233 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:18,223 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:18,413 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:21:18,413 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:21:18,414 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 with policy now +2025-01-06 17:21:18,414 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:21:18,414 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:21:18,415 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:21:18,631 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/6xakmz00-media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 +2025-01-06 17:21:18,824 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:21:18,824 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:21:18,824 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 +2025-01-06 17:21:18,824 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval +2025-01-06 17:21:19,078 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/fcewp0dk-media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 +2025-01-06 17:21:19,226 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:19,857 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:21:20,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:21:20,193 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:21:20,237 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:20,270 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:20,825 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:21:21,234 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:22,229 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:23,235 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:24,233 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:24,826 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:21:25,232 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:26,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:26,234 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:26,827 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:21:27,237 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:28,239 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:29,242 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:30,242 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:31,199 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:31,242 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:32,246 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:33,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:34,248 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:34,829 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:21:35,070 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/fjcxvmgt-events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:21:35,191 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:21:35,191 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:21:35,252 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:35,479 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:21:35,479 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:21:35,479 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:21:35,830 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:21:36,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:36,785 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:36,830 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:21:37,255 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:38,254 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:39,249 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:40,249 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:40,831 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:21:41,249 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:42,194 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:42,249 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:43,249 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:44,249 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:45,249 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:46,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:47,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:48,194 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:48,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:49,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:49,858 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:21:50,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:21:50,190 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:21:50,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:51,250 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:52,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:53,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:53,325 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:54,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:55,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:56,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:57,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:58,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:21:59,194 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:21:59,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:00,251 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:01,252 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:02,252 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:03,252 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:04,195 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:04,252 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:05,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:22:05,190 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:22:05,252 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:06,255 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:06,840 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:07,141 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:22:07,141 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 with policy now +2025-01-06 17:22:07,160 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:22:07,160 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:22:07,160 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:22:07,161 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:22:07,257 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:07,346 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/1lcx2tqf-media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 +2025-01-06 17:22:07,841 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:22:07,841 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 +2025-01-06 17:22:07,841 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval +2025-01-06 17:22:08,101 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/0pavcjfu-media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 +2025-01-06 17:22:08,261 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:08,841 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:09,262 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:09,841 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:10,192 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:10,261 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:11,259 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:12,267 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:12,842 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:13,264 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:14,262 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:14,843 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:15,197 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:15,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:16,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:17,267 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:18,263 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:19,261 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:19,530 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:22:19,530 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:22:19,530 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:22:19,845 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:22:19,845 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:22:19,858 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:22:20,193 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:22:20,193 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:22:20,265 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:20,341 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:20,845 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:21,267 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:22,265 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:22,846 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:23,268 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:24,265 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:24,847 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:25,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:26,196 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:26,265 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:26,847 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:27,265 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:28,265 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:29,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:30,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:31,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:32,196 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:32,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:33,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:34,266 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:35,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:22:35,190 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:22:35,267 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:36,267 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:37,240 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:37,267 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:38,267 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:39,268 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:40,268 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:41,268 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:42,268 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:43,196 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:43,268 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:44,268 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:45,269 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:46,269 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:47,269 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:48,197 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:48,269 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:49,269 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:49,859 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:22:50,190 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:22:50,190 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:22:50,269 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:51,273 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:51,912 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:22:51,913 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 with policy now +2025-01-06 17:22:51,917 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:22:51,918 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:22:51,918 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:22:51,918 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:22:52,130 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/9k26rq02-media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 +2025-01-06 17:22:52,272 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:52,856 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:22:52,857 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:52,857 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 +2025-01-06 17:22:52,857 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval +2025-01-06 17:22:53,129 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/x4lwqnba-media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 +2025-01-06 17:22:53,281 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:53,936 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:22:54,280 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:55,279 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:56,280 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:56,858 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:22:57,286 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:58,283 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:59,290 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:22:59,377 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:23:00,289 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:00,859 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:23:01,288 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:02,294 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:03,299 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:04,293 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:05,202 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:23:05,202 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:23:05,276 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:23:05,291 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:05,861 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:23:06,112 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/epgz5fhh-events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:23:06,280 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:23:06,280 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:23:06,294 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:06,460 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:23:06,725 INFO wandb-upload_0:2088223 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpt9ubxmqi +2025-01-06 17:23:06,809 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:23:06,822 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:23:06,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:23:06,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:23:06,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 17:23:06,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-06 17:23:06,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media +2025-01-06 17:23:06,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table +2025-01-06 17:23:06,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval +2025-01-06 17:23:07,102 INFO SenderThread:2088223 [sender.py:send_request_log_artifact():1525] logged artifact run-5w3hakcj-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTM0Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMDU5Mw==', 'latestArtifact': None}} +2025-01-06 17:23:07,102 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:23:07,102 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-06 17:23:07,104 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:23:07,104 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,104 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,104 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:23:07,105 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,108 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,108 DEBUG SenderThread:2088223 [sender.py:send():379] send: telemetry +2025-01-06 17:23:07,109 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,109 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,109 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,109 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,109 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,110 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,110 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,110 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,110 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,111 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,111 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,111 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,111 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,112 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,112 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,112 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,112 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,113 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,113 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,113 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,113 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,113 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,114 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:23:07,267 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/b8bdhkrj-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 17:23:07,290 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:07,324 INFO wandb-upload_1:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/kirrr60p-media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-06 17:23:07,484 INFO wandb-upload_0:2088223 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_goqsqj8 +2025-01-06 17:23:07,514 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/xs3f6cik-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 17:23:07,845 INFO SenderThread:2088223 [sender.py:send_request_log_artifact():1525] logged artifact run-5w3hakcj-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTM1OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMDU5NQ==', 'latestArtifact': None}} +2025-01-06 17:23:07,845 DEBUG SenderThread:2088223 [sender.py:send():379] send: files +2025-01-06 17:23:07,845 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json with policy now +2025-01-06 17:23:07,846 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:23:07,846 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:07,846 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:07,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:23:08,084 INFO wandb-upload_1:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/khasfmu7-media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-06 17:23:08,290 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:08,862 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:23:09,290 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:10,290 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:10,459 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:23:10,566 DEBUG SenderThread:2088223 [sender.py:send():379] send: exit +2025-01-06 17:23:10,567 INFO SenderThread:2088223 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 17:23:10,567 INFO SenderThread:2088223 [sender.py:send_exit():588] handling runtime: 260 +2025-01-06 17:23:10,568 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:10,568 INFO SenderThread:2088223 [sender.py:send_exit():594] send defer +2025-01-06 17:23:10,568 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:10,568 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 17:23:10,568 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:10,568 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 17:23:10,568 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 1 +2025-01-06 17:23:10,568 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:10,569 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 17:23:10,569 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:10,569 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 17:23:10,569 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 2 +2025-01-06 17:23:10,569 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:10,569 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 17:23:10,569 INFO HandlerThread:2088223 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 17:23:10,569 DEBUG SystemMonitor:2088223 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 17:23:10,569 DEBUG SystemMonitor:2088223 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 17:23:10,570 INFO HandlerThread:2088223 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 17:23:10,570 INFO HandlerThread:2088223 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 17:23:10,623 INFO HandlerThread:2088223 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 17:23:10,623 INFO HandlerThread:2088223 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 17:23:10,624 INFO HandlerThread:2088223 [interfaces.py:finish():200] Joined network monitor +2025-01-06 17:23:10,624 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:10,624 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 17:23:10,624 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 3 +2025-01-06 17:23:10,624 DEBUG SenderThread:2088223 [sender.py:send():379] send: stats +2025-01-06 17:23:10,624 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:10,624 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 17:23:10,625 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:10,625 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 17:23:10,625 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 4 +2025-01-06 17:23:10,625 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:10,625 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 17:23:10,863 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/config.yaml +2025-01-06 17:23:10,863 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:23:12,864 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:23:17,592 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:23:17,592 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:17,592 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:23:17,593 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 17:23:17,593 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 17:23:17,594 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 5 +2025-01-06 17:23:17,594 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:23:17,594 DEBUG SenderThread:2088223 [sender.py:send():379] send: history +2025-01-06 17:23:17,594 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:17,594 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 17:23:17,594 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:23:17,595 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:17,595 DEBUG SenderThread:2088223 [sender.py:send():379] send: summary +2025-01-06 17:23:17,596 INFO SenderThread:2088223 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:23:17,596 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:17,596 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 17:23:17,596 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 6 +2025-01-06 17:23:17,596 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:17,596 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 17:23:17,596 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:17,596 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 17:23:17,596 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 7 +2025-01-06 17:23:17,596 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:23:17,597 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:17,597 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 17:23:17,597 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:17,597 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 17:23:17,866 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:23:18,503 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:23:19,077 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 8 +2025-01-06 17:23:19,077 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:23:19,077 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:19,078 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 17:23:19,078 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:19,078 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 17:23:19,078 INFO SenderThread:2088223 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 17:23:19,080 INFO SenderThread:2088223 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 17:23:19,082 INFO SenderThread:2088223 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 17:23:19,090 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 9 +2025-01-06 17:23:19,091 DEBUG SenderThread:2088223 [sender.py:send():379] send: artifact +2025-01-06 17:23:19,093 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:19,093 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 17:23:19,450 INFO wandb-upload_1:2088223 [upload_job.py:push():85] Skipped uploading /tmp/tmpxdqw67u_/wandb-job.json +2025-01-06 17:23:19,450 INFO wandb-upload_0:2088223 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdz6bjyvd +2025-01-06 17:23:19,503 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:23:19,564 INFO wandb-upload_2:2088223 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp49htejn7 +2025-01-06 17:23:19,867 INFO Thread-12 :2088223 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:23:20,102 INFO SenderThread:2088223 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'versionIndex': 422}}} +2025-01-06 17:23:20,102 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:20,102 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 17:23:20,102 INFO SenderThread:2088223 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 17:23:20,868 INFO SenderThread:2088223 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files +2025-01-06 17:23:20,868 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/config.yaml config.yaml +2025-01-06 17:23:20,869 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json wandb-summary.json +2025-01-06 17:23:20,869 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:23:20,871 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/requirements.txt requirements.txt +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-metadata.json wandb-metadata.json +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log output.log +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/diff.patch diff.patch +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 media/videos/eval/1_eval_1_caf44f191ae0ae9f7dee.mp4 +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 media/videos/eval/0_eval_0_618aacdea903d28c41fe.mp4 +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 media/videos/eval/4_eval_4_4607322fe9bf46c96947.mp4 +2025-01-06 17:23:20,873 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 media/videos/eval/2_eval_2_493dcb44b239a781188a.mp4 +2025-01-06 17:23:20,874 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 media/videos/eval/3_eval_3_b633df55c1c901ea4269.mp4 +2025-01-06 17:23:20,874 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-06 17:23:20,875 INFO SenderThread:2088223 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 17:23:20,875 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 10 +2025-01-06 17:23:20,875 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:23:20,876 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:20,880 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 17:23:20,880 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:20,880 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 17:23:20,880 INFO SenderThread:2088223 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:23:21,039 INFO wandb-upload_2:2088223 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/wandb-summary.json +2025-01-06 17:23:21,056 INFO wandb-upload_1:2088223 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/config.yaml +2025-01-06 17:23:21,102 INFO wandb-upload_4:2088223 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/output.log +2025-01-06 17:23:21,120 INFO wandb-upload_0:2088223 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/files/requirements.txt +2025-01-06 17:23:21,143 INFO wandb-upload_3:2088223 [upload_job.py:push():130] Uploaded file /tmp/tmpg7sw7vxxwandb/aw4645rl-events.out.tfevents.1736201930.leonmkim-ROG-Strix-G15CS-G15CS.2088037.0 +2025-01-06 17:23:21,343 INFO Thread-11 (_thread_body):2088223 [sender.py:transition_state():614] send defer: 11 +2025-01-06 17:23:21,343 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:21,343 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 17:23:21,343 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:21,343 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 17:23:21,344 INFO SenderThread:2088223 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 17:23:21,345 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 12 +2025-01-06 17:23:21,345 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:21,345 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 17:23:21,345 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:21,345 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 17:23:21,345 INFO SenderThread:2088223 [file_stream.py:finish():601] file stream finish called +2025-01-06 17:23:21,406 INFO SenderThread:2088223 [file_stream.py:finish():605] file stream finish is done +2025-01-06 17:23:21,407 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 13 +2025-01-06 17:23:21,407 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:21,407 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 17:23:21,407 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:21,407 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 17:23:21,407 INFO SenderThread:2088223 [sender.py:transition_state():614] send defer: 14 +2025-01-06 17:23:21,407 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:23:21,407 INFO HandlerThread:2088223 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 17:23:21,407 DEBUG SenderThread:2088223 [sender.py:send():379] send: final +2025-01-06 17:23:21,408 DEBUG SenderThread:2088223 [sender.py:send():379] send: footer +2025-01-06 17:23:21,408 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: defer +2025-01-06 17:23:21,408 INFO SenderThread:2088223 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 17:23:21,408 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:23:21,408 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:23:21,408 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 17:23:21,409 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:23:21,409 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: server_info +2025-01-06 17:23:21,410 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 17:23:21,411 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 17:23:21,411 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:23:21,465 DEBUG SenderThread:2088223 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:23:21,465 INFO MainThread:2088223 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 17:23:21,465 INFO MainThread:2088223 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 17:23:21,466 INFO MainThread:2088223 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 17:23:21,466 DEBUG HandlerThread:2088223 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 17:23:21,466 INFO HandlerThread:2088223 [handler.py:finish():882] shutting down handler +2025-01-06 17:23:22,409 INFO WriterThread:2088223 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/run-5w3hakcj.wandb +2025-01-06 17:23:22,465 INFO SenderThread:2088223 [sender.py:finish():1615] shutting down sender +2025-01-06 17:23:22,465 INFO SenderThread:2088223 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:23:22,465 INFO SenderThread:2088223 [file_pusher.py:join():175] waiting for file pusher diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/logs/debug.log b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c25d9af0d9313f53f74b7a7b52b29f79c858cbb3 --- /dev/null +++ b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 17:18:49,528 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 17:18:49,528 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Configure stats pid to 2088037 +2025-01-06 17:18:49,528 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/settings +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/logs/debug.log +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/logs/debug-internal.log +2025-01-06 17:18:49,529 INFO MainThread:2088037 [wandb_init.py:init():569] calling init triggers +2025-01-06 17:18:49,530 INFO MainThread:2088037 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj'} +2025-01-06 17:18:49,530 INFO MainThread:2088037 [wandb_init.py:init():619] starting backend +2025-01-06 17:18:49,530 INFO MainThread:2088037 [wandb_init.py:init():623] setting up manager +2025-01-06 17:18:49,534 INFO MainThread:2088037 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 17:18:49,535 INFO MainThread:2088037 [wandb_init.py:init():631] backend started and connected +2025-01-06 17:18:49,547 INFO MainThread:2088037 [wandb_init.py:init():720] updated telemetry +2025-01-06 17:18:49,556 INFO MainThread:2088037 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 17:18:49,787 INFO MainThread:2088037 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 17:18:49,841 INFO MainThread:2088037 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 17:18:49,841 INFO MainThread:2088037 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 17:18:50,186 INFO MainThread:2088037 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 17:18:50,186 INFO MainThread:2088037 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 17:18:50,186 INFO MainThread:2088037 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 17:18:50,187 INFO MainThread:2088037 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 17:18:50,188 INFO MainThread:2088037 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 17:18:50,189 INFO MainThread:2088037 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5w3hakcj/tb, True +2025-01-06 17:18:59,898 INFO MainThread:2088037 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 17:23:22,468 WARNING MsgRouterThr:2088037 [router.py:message_loop():77] message_loop has been closed diff --git a/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/run-5w3hakcj.wandb b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/run-5w3hakcj.wandb new file mode 100644 index 0000000000000000000000000000000000000000..398e14f4280df240a8e7d31b2f515dbb0388b621 Binary files /dev/null and b/5w3hakcj/wandb/run-20250106_171849-5w3hakcj/run-5w3hakcj.wandb differ diff --git a/9fviy2t8/episode_rosbags/episode_1_2025-01-06-17-24-48.bag b/9fviy2t8/episode_rosbags/episode_1_2025-01-06-17-24-48.bag new file mode 100644 index 0000000000000000000000000000000000000000..45ce4677c71870c240470a5d4d3dcfb35dc2b693 --- /dev/null +++ b/9fviy2t8/episode_rosbags/episode_1_2025-01-06-17-24-48.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:854cfb9ff1435fcc982279d95385ebc775ff1c60dffc8c7cf857e347db4464b5 +size 1413075265 diff --git a/9fviy2t8/episode_rosbags/episode_2_2025-01-06-17-25-35.bag b/9fviy2t8/episode_rosbags/episode_2_2025-01-06-17-25-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..22505cbdc1250c79b6f5d96e6f838f01a5bd1bdb --- /dev/null +++ b/9fviy2t8/episode_rosbags/episode_2_2025-01-06-17-25-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea48adf59f9f97e9d1eed43ba5a9bf9b8db30b4caf6ca5c61c6664a736b1a21e +size 1408321492 diff --git a/9ig9megf/episode_rosbags/episode_0_2025-01-06-22-14-04.bag b/9ig9megf/episode_rosbags/episode_0_2025-01-06-22-14-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..3e8520a63a7eb0a0d6b06efe3df40dc2570a3960 --- /dev/null +++ b/9ig9megf/episode_rosbags/episode_0_2025-01-06-22-14-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8847791365c3f11b2b720ccbf8b22ccd07bc748b7f525b61d31e13ccf47f8cd0 +size 1408100745 diff --git a/9ig9megf/episode_rosbags/episode_1_2025-01-06-22-14-52.bag b/9ig9megf/episode_rosbags/episode_1_2025-01-06-22-14-52.bag new file mode 100644 index 0000000000000000000000000000000000000000..951c68e5571d1e7092023ab87086bacabcae9a77 --- /dev/null +++ b/9ig9megf/episode_rosbags/episode_1_2025-01-06-22-14-52.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46ca728bebe0b28f74a54fd9893a78b3b1310090788d061bce0a08bb02382048 +size 1412121787 diff --git a/9ig9megf/episode_rosbags/episode_2_2025-01-06-22-15-35.bag b/9ig9megf/episode_rosbags/episode_2_2025-01-06-22-15-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..407f29b6521d9f281523d0efe2f9cd21bf3c245a --- /dev/null +++ b/9ig9megf/episode_rosbags/episode_2_2025-01-06-22-15-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:892d998756cf7567d095b5f91f02a958ac3a301c76f27ae6256afbe294059cba +size 1410647536 diff --git a/9ig9megf/episode_rosbags/episode_3_2025-01-06-22-16-15.bag b/9ig9megf/episode_rosbags/episode_3_2025-01-06-22-16-15.bag new file mode 100644 index 0000000000000000000000000000000000000000..9e99327eee16167fd54638a38b28998f40902876 --- /dev/null +++ b/9ig9megf/episode_rosbags/episode_3_2025-01-06-22-16-15.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34d46206967388ba9a70e0980d0a568d04cca75166707048681a9d0f4786afd0 +size 1412320489 diff --git a/em58uo70/.hydra/config.yaml b/em58uo70/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab --- /dev/null +++ b/em58uo70/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1009_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/em58uo70/.hydra/hydra.yaml b/em58uo70/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..822df5ce296870f2ae9a7085c10ed44e18d9e82f --- /dev/null +++ b/em58uo70/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/em58uo70/.hydra/overrides.yaml b/em58uo70/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/em58uo70/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/em58uo70/episode_rosbags/aligned_depth_to_color_K.npy b/em58uo70/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/em58uo70/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/em58uo70/episode_rosbags/cam_tf_world.npy b/em58uo70/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/em58uo70/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/em58uo70/episode_rosbags/color_K.npy b/em58uo70/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/em58uo70/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/em58uo70/episode_rosbags/depth_K.npy b/em58uo70/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/em58uo70/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/em58uo70/eval_robot.log b/em58uo70/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..b9101f104bb4a5e905d28aac53263cffd2920e47 --- /dev/null +++ b/em58uo70/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 17:50:40,361][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 17:50:40,364][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/em58uo70/eval_video/1_eval.mp4 b/em58uo70/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c5d75d2b84422c191bf9a7cb14cfea1e15cfcd52 --- /dev/null +++ b/em58uo70/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b5bcc7391582f2bd11469686e27dd185745aaee20a4695c4fb61f09c3bed51b +size 1136231 diff --git a/em58uo70/eval_video/3_eval.mp4 b/em58uo70/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..931bf63bf02a7825ef222e0d8e904f4e547f0012 --- /dev/null +++ b/em58uo70/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e5f95db56c8842b2074898009377c6f5b1e39e7116f9e62e5353df63532f0e1 +size 1133816 diff --git a/em58uo70/eval_video/4_eval.mp4 b/em58uo70/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6da7dbf042393678fb99f81c6441cec637896755 --- /dev/null +++ b/em58uo70/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b0616a340d3be9a0adda599d1c3d2320f10eead7ccf080832ba674ef124cedc +size 1196708 diff --git a/em58uo70/tb/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 b/em58uo70/tb/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 new file mode 100644 index 0000000000000000000000000000000000000000..6d5f69fbaa28f6e9810ad6e7f50527b4c647f77b --- /dev/null +++ b/em58uo70/tb/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4798daad6405786c9f91f9ad2159eda2a6fa8e01124cf44f9aa151280da6e12c +size 6539 diff --git a/em58uo70/wandb/debug-internal.log b/em58uo70/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..47e85cf593432f19866e67386535287ba26a516a --- /dev/null +++ b/em58uo70/wandb/debug-internal.log @@ -0,0 +1,727 @@ +2025-01-07 17:50:46,219 INFO StreamThr :2204041 [internal.py:wandb_internal():85] W&B internal server running at pid: 2204041, started at: 2025-01-07 17:50:46.219571 +2025-01-07 17:50:46,220 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:50:46,232 INFO WriterThread:2204041 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/run-em58uo70.wandb +2025-01-07 17:50:46,232 DEBUG SenderThread:2204041 [sender.py:send():379] send: header +2025-01-07 17:50:46,245 DEBUG SenderThread:2204041 [sender.py:send():379] send: run +2025-01-07 17:50:46,510 INFO SenderThread:2204041 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files +2025-01-07 17:50:46,510 INFO SenderThread:2204041 [sender.py:_start_run_threads():1188] run started: em58uo70 with start time 1736290246.219841 +2025-01-07 17:50:46,527 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:50:46,527 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:50:46,603 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:50:46,608 DEBUG HandlerThread:2204041 [system_info.py:__init__():26] System info init +2025-01-07 17:50:46,608 DEBUG HandlerThread:2204041 [system_info.py:__init__():41] System info init done +2025-01-07 17:50:46,608 INFO HandlerThread:2204041 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:50:46,608 INFO SystemMonitor:2204041 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:50:46,608 INFO HandlerThread:2204041 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:50:46,609 INFO SystemMonitor:2204041 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:50:46,609 INFO SystemMonitor:2204041 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:50:46,610 INFO SystemMonitor:2204041 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:50:46,611 INFO SystemMonitor:2204041 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:50:46,611 INFO SystemMonitor:2204041 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:50:46,621 DEBUG HandlerThread:2204041 [system_info.py:probe():152] Probing system +2025-01-07 17:50:46,622 DEBUG HandlerThread:2204041 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:50:46,627 DEBUG HandlerThread:2204041 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:50:46,627 DEBUG HandlerThread:2204041 [system_info.py:probe():200] Probing system done +2025-01-07 17:50:46,627 DEBUG HandlerThread:2204041 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:50:46.621332', 'startedAt': '2025-01-07T22:50:46.206437', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.8066249999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.565, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.989, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 783.1284484863281}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:50:46,627 INFO HandlerThread:2204041 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:50:46,627 INFO HandlerThread:2204041 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:50:46,627 DEBUG HandlerThread:2204041 [system_info.py:_save_code():44] Saving code +2025-01-07 17:50:46,633 DEBUG HandlerThread:2204041 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:50:46,633 DEBUG HandlerThread:2204041 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:50:46,700 DEBUG HandlerThread:2204041 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:50:46,700 INFO HandlerThread:2204041 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:50:46,704 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:50:46,704 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:50:46,704 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:50:46,704 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:50:46,945 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/lchhyf4v-wandb-metadata.json +2025-01-07 17:50:46,958 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:50:46,959 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:50:46,959 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:50:46,959 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:50:46,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:46,962 INFO HandlerThread:2204041 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/tb" + save: true +} +control { + relay_id: "9d3497cdfa604630b5ad827e26575f59" +} +_info { + stream_id: "em58uo70" +} + +2025-01-07 17:50:46,962 DEBUG HandlerThread:2204041 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:50:47,001 INFO wandb-upload_2:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/eeyox32h-diff.patch +2025-01-07 17:50:47,011 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/2htrlx2o-code/FISH/eval_robot.py +2025-01-07 17:50:47,070 DEBUG SenderThread:2204041 [sender.py:send():379] send: telemetry +2025-01-07 17:50:47,070 DEBUG SenderThread:2204041 [sender.py:send():379] send: tbrecord +2025-01-07 17:50:47,071 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:50:47,071 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 with policy live +2025-01-07 17:50:47,266 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/kt5i0mta-events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:50:47,512 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/code/FISH/eval_robot.py +2025-01-07 17:50:47,512 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/requirements.txt +2025-01-07 17:50:47,512 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:50:47,512 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-metadata.json +2025-01-07 17:50:47,512 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:50:47,512 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/diff.patch +2025-01-07 17:50:47,513 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/code +2025-01-07 17:50:47,513 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/code/FISH +2025-01-07 17:50:47,959 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:48,959 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:49,511 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:50:49,959 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:50,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:51,303 DEBUG SenderThread:2204041 [sender.py:send():379] send: config +2025-01-07 17:50:51,304 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:50:51,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:52,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:53,512 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:50:53,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:54,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:55,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:56,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:50:56,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:57,513 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:50:57,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:58,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:59,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:00,977 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:01,514 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:01,959 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:51:01,959 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:51:02,005 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:02,038 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:02,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:03,515 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:03,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:04,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:05,516 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:05,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:06,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:07,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:07,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:08,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:09,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:10,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:11,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:12,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:12,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:13,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:14,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:15,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:16,959 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:51:16,959 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:51:17,005 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:17,963 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:18,092 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:18,519 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/config.yaml +2025-01-07 17:51:18,963 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:19,963 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:20,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:21,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:22,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:23,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:23,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:24,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:25,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:26,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:27,521 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:27,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:28,965 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:29,089 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:29,522 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:29,965 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:30,965 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:31,523 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:31,576 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:51:31,576 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 with policy now +2025-01-07 17:51:31,576 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:51:31,577 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:51:31,577 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:51:31,577 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:51:31,839 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/6tp6y3yi-media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 +2025-01-07 17:51:31,959 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:51:31,959 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:51:31,965 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:32,523 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:51:32,524 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 +2025-01-07 17:51:32,524 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval +2025-01-07 17:51:32,525 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos +2025-01-07 17:51:32,525 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media +2025-01-07 17:51:32,830 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/zs22w709-media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 +2025-01-07 17:51:32,965 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:33,524 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:33,966 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:34,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:35,055 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:35,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:36,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:37,525 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:37,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:38,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:39,526 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:39,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:40,599 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:40,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:41,527 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:51:41,527 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:41,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:42,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:43,527 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:43,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:44,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:45,528 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:45,881 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:45,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:46,612 DEBUG SystemMonitor:2204041 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:51:46,663 DEBUG SenderThread:2204041 [sender.py:send():379] send: stats +2025-01-07 17:51:46,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:51:46,960 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:51:47,005 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:47,529 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:47,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:48,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:49,530 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:49,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:50,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:51,525 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:51,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:52,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:53,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:54,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:55,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:56,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:56,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:57,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:58,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:59,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:00,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:01,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:52:01,960 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:52:02,001 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:02,047 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:02,970 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:03,970 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:04,970 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:05,970 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:06,970 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:07,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:07,971 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:08,971 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:09,971 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:10,971 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:11,536 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:11,971 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:12,773 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:52:12,773 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 with policy now +2025-01-07 17:52:12,776 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:52:12,776 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:52:12,776 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:52:12,777 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:52:12,971 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:13,200 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/4n6po6lp-media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 +2025-01-07 17:52:13,537 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:13,537 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:52:13,537 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 +2025-01-07 17:52:13,538 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval +2025-01-07 17:52:13,802 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:13,828 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/na48ggjt-media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 +2025-01-07 17:52:13,972 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:14,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:15,538 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:15,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:16,613 DEBUG SenderThread:2204041 [sender.py:send():379] send: stats +2025-01-07 17:52:16,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:52:16,962 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:52:16,977 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:17,539 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:17,979 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:18,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:19,119 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:19,979 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:20,972 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:21,540 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:21,972 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:22,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:23,348 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:52:23,349 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:52:23,349 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:52:23,541 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:52:23,541 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:52:23,806 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/f8ukn2cw-events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:52:23,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:24,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:25,101 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:25,541 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:25,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:26,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:27,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:28,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:29,543 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:29,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:30,175 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:30,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:31,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:52:31,961 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:52:32,001 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:32,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:33,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:34,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:35,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:36,057 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:36,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:37,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:38,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:39,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:40,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:41,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:41,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:42,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:43,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:44,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:45,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:46,613 DEBUG SenderThread:2204041 [sender.py:send():379] send: stats +2025-01-07 17:52:46,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:52:46,961 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:52:47,001 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:47,090 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:47,976 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:48,976 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:49,977 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:50,977 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:51,978 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:52,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:52,978 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:53,978 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:54,979 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:55,081 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:52:55,081 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:52:55,082 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 with policy now +2025-01-07 17:52:55,082 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:52:55,082 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:52:55,083 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:52:55,351 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/da1jldbk-media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 +2025-01-07 17:52:55,551 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:55,552 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:52:55,552 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 +2025-01-07 17:52:55,552 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval +2025-01-07 17:52:55,828 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/33c2ull0-media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 +2025-01-07 17:52:55,985 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:56,985 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:57,552 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:57,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:57,989 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:58,982 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:59,987 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:00,985 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:01,553 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:01,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:53:01,961 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:53:01,984 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:02,993 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:03,119 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:03,554 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:03,986 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:04,989 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:05,183 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:53:05,183 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:53:05,184 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:53:05,555 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:53:05,555 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:05,555 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:53:05,989 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:06,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:07,555 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:07,988 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:08,700 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:08,990 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:09,556 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:09,996 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:10,996 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:11,557 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:11,996 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:12,996 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:13,970 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:13,996 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:14,996 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:15,996 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:16,614 DEBUG SenderThread:2204041 [sender.py:send():379] send: stats +2025-01-07 17:53:16,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:53:16,961 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:53:17,001 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:17,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:18,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:19,038 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:19,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:20,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:21,560 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:21,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:22,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:23,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:24,970 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:24,998 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:25,998 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:26,998 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:27,998 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:28,998 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:29,971 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:29,999 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:30,999 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:31,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:53:31,961 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:53:32,005 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:32,999 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:33,999 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:34,999 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:35,100 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:36,003 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:36,981 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:53:36,981 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 with policy now +2025-01-07 17:53:36,984 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:53:36,984 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:53:36,984 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:53:36,985 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:53:37,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:37,251 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/699weixh-media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 +2025-01-07 17:53:37,566 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:37,566 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:53:37,566 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 +2025-01-07 17:53:37,566 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval +2025-01-07 17:53:37,850 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/dna5w348-media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 +2025-01-07 17:53:38,006 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:39,011 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:39,566 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:40,006 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:40,727 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:41,027 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:41,567 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:42,019 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:43,014 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:44,009 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:45,018 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:45,568 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:46,020 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:46,614 DEBUG SenderThread:2204041 [sender.py:send():379] send: stats +2025-01-07 17:53:46,614 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:46,963 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:53:46,964 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:53:47,021 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:48,020 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:49,022 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:50,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:50,239 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:53:50,239 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:53:50,239 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:53:50,570 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:53:50,570 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:53:50,867 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/x5t4ko3i-events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:53:51,022 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:51,570 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:51,979 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:52,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:53,021 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:53,571 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:54,027 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:55,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:55,572 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:56,023 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:57,023 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:57,573 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:57,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:58,023 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:59,023 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:00,023 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:01,023 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:01,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:54:01,962 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:54:02,024 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:03,024 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:03,082 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:04,024 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:05,024 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:06,024 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:07,024 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:08,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:08,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:09,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:10,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:11,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:12,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:13,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:13,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:14,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:15,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:16,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:16,615 DEBUG SenderThread:2204041 [sender.py:send():379] send: stats +2025-01-07 17:54:16,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:54:16,962 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:54:17,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:18,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:19,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:19,101 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:20,027 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:21,027 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:22,027 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:23,031 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:23,582 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:23,927 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:54:23,927 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 with policy now +2025-01-07 17:54:23,939 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:54:23,939 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:54:23,940 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:23,940 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:24,033 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:24,186 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/rb8yjj6w-media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 +2025-01-07 17:54:24,582 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:54:24,582 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 +2025-01-07 17:54:24,582 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval +2025-01-07 17:54:24,913 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/7f41e2bk-media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 +2025-01-07 17:54:24,977 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:25,037 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:25,582 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:26,040 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:27,039 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:28,041 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:29,044 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:29,584 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:30,050 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:30,520 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:31,074 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:31,584 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:31,966 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:54:31,967 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:54:32,054 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:33,066 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:34,055 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:34,308 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:54:34,309 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:34,310 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:34,541 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:54:34,542 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:54:34,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:54:34,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:54:34,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:54:34,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media +2025-01-07 17:54:34,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval +2025-01-07 17:54:34,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table +2025-01-07 17:54:34,727 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:54:34,784 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/wspzzxuq-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:54:35,031 INFO wandb-upload_0:2204041 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyp8pg8x3 +2025-01-07 17:54:35,055 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:35,113 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:54:35,157 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:54:35,453 INFO SenderThread:2204041 [sender.py:send_request_log_artifact():1525] logged artifact run-em58uo70-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjAxNzgzNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcyNzQ3MQ==', 'latestArtifact': None}} +2025-01-07 17:54:35,453 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:54:35,453 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-07 17:54:35,453 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:54:35,453 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,454 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,454 DEBUG SenderThread:2204041 [sender.py:send():379] send: telemetry +2025-01-07 17:54:35,454 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,458 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,458 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,458 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,458 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,459 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,459 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,459 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,459 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,459 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,460 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,460 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,460 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,460 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,461 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,461 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,461 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,461 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,462 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,462 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,462 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,462 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,463 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:54:35,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:54:35,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:35,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:54:35,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json +2025-01-07 17:54:35,587 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval +2025-01-07 17:54:35,719 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/kvw6i9vs-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:54:35,774 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/uax9iuap-media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json +2025-01-07 17:54:35,845 INFO wandb-upload_2:2204041 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpx_qywsap +2025-01-07 17:54:36,055 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:36,240 INFO SenderThread:2204041 [sender.py:send_request_log_artifact():1525] logged artifact run-em58uo70-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjAxNzg2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcyNzQ3Ng==', 'latestArtifact': None}} +2025-01-07 17:54:36,245 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:36,387 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:54:36,387 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json with policy now +2025-01-07 17:54:36,388 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:54:36,388 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:36,388 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:36,587 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/config.yaml +2025-01-07 17:54:36,587 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:54:36,655 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/c4mmi5oo-media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json +2025-01-07 17:54:37,055 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:37,587 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:38,055 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:39,056 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:40,056 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:40,925 DEBUG SenderThread:2204041 [sender.py:send():379] send: exit +2025-01-07 17:54:40,926 INFO SenderThread:2204041 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:54:40,926 INFO SenderThread:2204041 [sender.py:send_exit():588] handling runtime: 234 +2025-01-07 17:54:40,926 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:40,926 INFO SenderThread:2204041 [sender.py:send_exit():594] send defer +2025-01-07 17:54:40,926 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:40,926 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:54:40,926 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:40,927 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:54:40,927 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:54:40,927 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:40,927 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:54:40,927 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:40,927 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:54:40,927 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:54:40,927 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:40,927 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:54:40,927 INFO HandlerThread:2204041 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:54:40,927 DEBUG SystemMonitor:2204041 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:54:40,927 DEBUG SystemMonitor:2204041 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:54:40,928 INFO HandlerThread:2204041 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:54:40,928 INFO HandlerThread:2204041 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:54:40,997 INFO HandlerThread:2204041 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:54:40,997 INFO HandlerThread:2204041 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:54:40,997 INFO HandlerThread:2204041 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:54:40,998 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:40,998 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:54:40,998 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:54:40,998 DEBUG SenderThread:2204041 [sender.py:send():379] send: stats +2025-01-07 17:54:40,998 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:40,998 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:54:40,999 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:40,999 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:54:40,999 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:54:40,999 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:40,999 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:54:41,589 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:54:43,590 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:47,315 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:54:47,315 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:47,315 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:54:47,315 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:47,317 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:47,317 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:54:47,317 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:54:47,317 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:54:47,317 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:47,317 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:54:47,317 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:54:47,317 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:47,319 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:47,319 DEBUG SenderThread:2204041 [sender.py:send():379] send: summary +2025-01-07 17:54:47,323 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:47,323 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:47,323 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:54:47,323 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:54:47,323 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:47,323 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:54:47,324 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:47,324 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:54:47,324 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:54:47,324 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:47,324 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:47,324 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:54:47,324 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:47,324 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:54:47,591 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:54:47,927 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:54:50,723 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:54:50,723 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:54:50,724 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:50,724 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:54:50,724 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:50,724 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:54:50,724 INFO SenderThread:2204041 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:54:50,726 INFO SenderThread:2204041 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:54:50,728 INFO SenderThread:2204041 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:54:50,742 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:54:50,743 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:50,743 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:54:50,743 DEBUG SenderThread:2204041 [sender.py:send():379] send: artifact +2025-01-07 17:54:50,928 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:54:50,935 INFO SenderThread:2204041 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 17:54:50,936 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:50,936 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:54:50,936 INFO SenderThread:2204041 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:54:51,592 INFO SenderThread:2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:51,593 INFO SenderThread:2204041 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files +2025-01-07 17:54:51,593 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/config.yaml config.yaml +2025-01-07 17:54:51,593 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json wandb-summary.json +2025-01-07 17:54:51,593 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/requirements.txt requirements.txt +2025-01-07 17:54:51,593 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:54:51,593 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:54:51,594 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log output.log +2025-01-07 17:54:51,594 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/diff.patch diff.patch +2025-01-07 17:54:51,598 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:54:51,599 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 +2025-01-07 17:54:51,599 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 +2025-01-07 17:54:51,599 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 +2025-01-07 17:54:51,599 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 +2025-01-07 17:54:51,599 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 +2025-01-07 17:54:51,601 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:54:51,601 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json +2025-01-07 17:54:51,601 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:54:51,602 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:54:51,603 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:51,603 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:54:51,603 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:51,603 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:54:51,603 INFO SenderThread:2204041 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:54:51,786 INFO wandb-upload_2:2204041 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/config.yaml +2025-01-07 17:54:51,795 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/requirements.txt +2025-01-07 17:54:51,820 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:54:51,823 INFO wandb-upload_4:2204041 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:51,906 INFO wandb-upload_3:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/v587226h-events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:54:51,928 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:54:51,928 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:54:52,106 INFO Thread-11 (_thread_body):2204041 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:54:52,106 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:52,106 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:54:52,106 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:52,106 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:54:52,107 INFO SenderThread:2204041 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:54:52,108 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:54:52,108 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:52,108 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:54:52,108 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:52,108 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:54:52,108 INFO SenderThread:2204041 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:54:52,305 INFO SenderThread:2204041 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:54:52,305 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:54:52,305 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:52,305 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:54:52,305 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:52,305 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:54:52,306 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:54:52,306 DEBUG SenderThread:2204041 [sender.py:send():379] send: final +2025-01-07 17:54:52,306 DEBUG SenderThread:2204041 [sender.py:send():379] send: footer +2025-01-07 17:54:52,306 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:52,306 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:54:52,306 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:52,306 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:54:52,307 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:54:52,307 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:54:52,307 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:54:52,307 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:54:52,309 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:54:52,309 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:54:52,310 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:54:52,311 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:52,364 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:54:52,365 INFO MainThread:2204041 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:54:52,365 INFO MainThread:2204041 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:54:52,365 INFO MainThread:2204041 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:54:52,366 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:54:52,366 INFO HandlerThread:2204041 [handler.py:finish():882] shutting down handler +2025-01-07 17:54:53,309 INFO WriterThread:2204041 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/run-em58uo70.wandb +2025-01-07 17:54:53,364 INFO SenderThread:2204041 [sender.py:finish():1615] shutting down sender +2025-01-07 17:54:53,365 INFO SenderThread:2204041 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:54:53,365 INFO SenderThread:2204041 [file_pusher.py:join():175] waiting for file pusher diff --git a/em58uo70/wandb/debug.log b/em58uo70/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e54204a772cd325e62e0dd0a9b8697906bc0e040 --- /dev/null +++ b/em58uo70/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Configure stats pid to 2203862 +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/settings +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/logs/debug.log +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/logs/debug-internal.log +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70'} +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_init.py:init():619] starting backend +2025-01-07 17:50:46,214 INFO MainThread:2203862 [wandb_init.py:init():623] setting up manager +2025-01-07 17:50:46,218 INFO MainThread:2203862 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:50:46,219 INFO MainThread:2203862 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:50:46,231 INFO MainThread:2203862 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:50:46,241 INFO MainThread:2203862 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:50:46,526 INFO MainThread:2203862 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:50:46,596 INFO MainThread:2203862 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:50:46,597 INFO MainThread:2203862 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:50:46,959 INFO MainThread:2203862 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:50:46,959 INFO MainThread:2203862 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:50:46,959 INFO MainThread:2203862 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:50:46,959 INFO MainThread:2203862 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:50:46,961 INFO MainThread:2203862 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:50:46,961 INFO MainThread:2203862 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/tb, True +2025-01-07 17:50:51,303 INFO MainThread:2203862 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 17:54:53,366 WARNING MsgRouterThr:2203862 [router.py:message_loop():77] message_loop has been closed diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/files/code/FISH/eval_robot.py b/em58uo70/wandb/run-20250107_175046-em58uo70/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/files/config.yaml b/em58uo70/wandb/run-20250107_175046-em58uo70/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09d2c94b80eec9796de576cc61213fcdb4b85276 --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1009_1' +wandb_notes: + desc: null + value: 1009_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736290246 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/files/diff.patch b/em58uo70/wandb/run-20250107_175046-em58uo70/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c0161c26d9b20c7c85242592d6833c2df9767742 --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/files/diff.patch @@ -0,0 +1,20 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..aa5b9b9 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,13 +134,13 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' +-# wandb_run_id: '1009_1' # seed 1 ++wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json b/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json new file mode 100644 index 0000000000000000000000000000000000000000..af5298d5b3bd51d71040535cae8addcbef63f42f --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[50, 50, 1.0, 0.9978451429801887, 1, 0.9350126980865203, 1, 0.9438994674986911, 1, 1, 1]]} \ No newline at end of file diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 b/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3fd24f8daff4f0b381b40aacba1862d26944a97b --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:424dad725925d47367ef4841880d62f929c2f12cbb49df4ab8eb6cc05831374e +size 1138064 diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 b/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c5d75d2b84422c191bf9a7cb14cfea1e15cfcd52 --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b5bcc7391582f2bd11469686e27dd185745aaee20a4695c4fb61f09c3bed51b +size 1136231 diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 b/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..019f073b4c6eb2eefdbfe51ca5d4486f1f79f8f7 --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2809d134000f06b2ecbc2350c5a4781efd46746d2d793b6d50edfe08b1c6047c +size 1138373 diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 b/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..931bf63bf02a7825ef222e0d8e904f4e547f0012 --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e5f95db56c8842b2074898009377c6f5b1e39e7116f9e62e5353df63532f0e1 +size 1133816 diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 b/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6da7dbf042393678fb99f81c6441cec637896755 --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b0616a340d3be9a0adda599d1c3d2320f10eead7ccf080832ba674ef124cedc +size 1196708 diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log b/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..58e419c8d2252c3e24e9aea381c1e4f30af4f506 --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log @@ -0,0 +1,286 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736290251.304024]: resetting environment +[INFO] [1736290251.307621]: cleared current plan +[INFO] [1736290251.308615]: moving to home +[INFO] [1736290252.311372]: reached home +[INFO] [1736290252.312708]: reset action history +[INFO] [1736290255.021211]: environment reset +Starting episode 0 +[INFO] [1736290255.023184]: resetting environment +[INFO] [1736290255.027667]: cleared current plan +[INFO] [1736290255.028603]: moving to home +[INFO] [1736290256.030861]: reached home +[INFO] [1736290256.032181]: reset action history +[INFO] [1736290258.740925]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736290259.927745]: resetting environment +[INFO] [1736290259.930574]: cleared current plan +[INFO] [1736290259.931347]: moving to home +[INFO] [1736290260.933270]: reached home +[INFO] [1736290260.934709]: reset action history +[INFO] [1736290263.641807]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/episode_rosbags/episode_0_2025-01-07-17-51-03.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2203862.log +started roslaunch server http://158.130.50.37:37419/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2204139] +process[print_text-2]: started with pid [2204140] +process[rosbag_record-3]: started with pid [2204164] +started bagging! +For topic gripper_width: timestamp difference is 100008228 for nearest: 1736290285245513656 - target: 1736290285145505428 at idx 249 +For topic gripper_width: timestamp difference is 100099411 for nearest: 1736290285445444431 - target: 1736290285345345020 at idx 249 +For topic gripper_width: timestamp difference is 100758321 for nearest: 1736290286245462186 - target: 1736290286144703865 at idx 249 +For topic gripper_width: timestamp difference is 101294947 for nearest: 1736290286845517588 - target: 1736290286744222641 at idx 249 +For topic gripper_width: timestamp difference is 101425621 for nearest: 1736290287045487854 - target: 1736290286944062233 at idx 249 +For topic gripper_width: timestamp difference is 104051105 for nearest: 1736290290245489351 - target: 1736290290141438246 at idx 249 +For topic gripper_width: timestamp difference is 104222228 for nearest: 1736290290445490052 - target: 1736290290341267824 at idx 249 +[INFO] [1736290290.616740]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736290291.372790]: Stored episode 1. +[INFO] [1736290291.373140]: Saving video... +[INFO] [1736290291.560550]: Video saved! +Starting episode 1 +[INFO] [1736290291.578206]: resetting environment +[INFO] [1736290291.584836]: cleared current plan +[INFO] [1736290291.585067]: moving to home +[INFO] [1736290295.288015]: reached home +[INFO] [1736290295.288722]: reset action history +[INFO] [1736290297.994363]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736290300.603216]: resetting environment +[INFO] [1736290300.606975]: cleared current plan +[INFO] [1736290300.607272]: moving to home +[INFO] [1736290301.608891]: reached home +[INFO] [1736290301.609312]: reset action history +[INFO] [1736290304.316481]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/episode_rosbags/episode_1_2025-01-07-17-51-44.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2203862.log +started roslaunch server http://158.130.50.37:45423/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2204285] +process[print_text-5]: started with pid [2204286] +process[rosbag_record-6]: started with pid [2204287] +started bagging! +For topic gripper_width: timestamp difference is 116599877 for nearest: 1736290305878873665 - target: 1736290305762273788 at idx 22 +For topic gripper_width: timestamp difference is 118715578 for nearest: 1736290308478821093 - target: 1736290308360105515 at idx 54 +For topic gripper_width: timestamp difference is 102260648 for nearest: 1736290329278820811 - target: 1736290329176560163 at idx 249 +For topic gripper_width: timestamp difference is 102428043 for nearest: 1736290329478824699 - target: 1736290329376396656 at idx 249 +For topic gripper_width: timestamp difference is 102611441 for nearest: 1736290329678844590 - target: 1736290329576233149 at idx 249 +[INFO] [1736290331.252073]: Storing episode... +For topic gripper_width: timestamp difference is 103903449 for nearest: 1736290331278820908 - target: 1736290331174917459 at idx 249 +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1736290332.054214]: Stored episode 2. +[INFO] [1736290332.055582]: Saving video... +[WARN] [1736290332.577318]: Plan exhausted +[WARN] [1736290332.616875]: Plan exhausted +[WARN] [1736290332.658236]: Plan exhausted +[WARN] [1736290332.697399]: Plan exhausted +[WARN] [1736290332.736870]: Plan exhausted +[WARN] [1736290332.776847]: Plan exhausted +[INFO] [1736290332.747107]: Video saved! +Starting episode 2 +[INFO] [1736290332.779892]: resetting environment +[INFO] [1736290332.801966]: cleared current plan +[INFO] [1736290332.802183]: moving to home +[INFO] [1736290336.506214]: reached home +[INFO] [1736290336.514583]: reset action history +[INFO] [1736290339.289820]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736290343.046042]: resetting environment +[INFO] [1736290343.056262]: cleared current plan +[INFO] [1736290343.056832]: moving to home +[INFO] [1736290344.060506]: reached home +[INFO] [1736290344.060995]: reset action history +[INFO] [1736290346.767063]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/episode_rosbags/episode_2_2025-01-07-17-52-26.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2203862.log +started roslaunch server http://158.130.50.37:39769/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2204446] +process[print_text-8]: started with pid [2204447] +process[rosbag_record-9]: started with pid [2204448] +started bagging! +[INFO] [1736290373.885451]: Storing episode... +[pub_text-7] killing on exit +[print_text-8] killing on exit[rosbag_record-9] killing on exit +[INFO] [1736290374.669806]: Stored episode 3. +[INFO] [1736290374.670478]: Saving video... +[INFO] [1736290375.035666]: Video saved! +Starting episode 3 +[INFO] [1736290375.076225]: resetting environment +[INFO] [1736290375.094340]: cleared current plan +[INFO] [1736290375.098775]: moving to home +[INFO] [1736290378.802081]: reached home +[INFO] [1736290378.813791]: reset action history +[INFO] [1736290381.654210]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736290384.719055]: resetting environment +[INFO] [1736290384.734087]: cleared current plan +[INFO] [1736290384.735271]: moving to home +[INFO] [1736290385.740068]: reached home +[INFO] [1736290385.744937]: reset action history +[INFO] [1736290388.699584]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/episode_rosbags/episode_3_2025-01-07-17-53-08.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2203862.log +started roslaunch server http://158.130.50.37:33717/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2204557] +process[print_text-11]: started with pid [2204558] +process[rosbag_record-12]: started with pid [2204559] +started bagging! +For topic gripper_width: timestamp difference is -180372773 for nearest: 1736290399178849162 - target: 1736290399359221935 at idx 154 +For topic gripper_width: timestamp difference is -112867414 for nearest: 1736290399678826273 - target: 1736290399791693687 at idx 158 +[INFO] [1736290415.647087]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit[print_text-11] killing on exit +[INFO] [1736290416.456556]: Stored episode 4. +[INFO] [1736290416.461948]: Saving video... +[WARN] [1736290416.992748]: Plan exhausted +[INFO] [1736290416.931044]: Video saved! +Starting episode 4 +[INFO] [1736290416.988147]: resetting environment +[INFO] [1736290417.005571]: cleared current plan +[INFO] [1736290417.006479]: moving to home +[INFO] [1736290420.723426]: reached home +[INFO] [1736290420.730743]: reset action history +[INFO] [1736290423.586956]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736290430.242344]: resetting environment +[INFO] [1736290430.264742]: cleared current plan +[INFO] [1736290430.274086]: moving to home +[INFO] [1736290431.288615]: reached home +[INFO] [1736290431.297636]: reset action history +[INFO] [1736290434.071346]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/episode_rosbags/episode_4_2025-01-07-17-53-54.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2203862.log +started roslaunch server http://158.130.50.37:39683/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2204716] +process[print_text-14]: started with pid [2204717] +process[rosbag_record-15]: started with pid [2204718] +started bagging! +[INFO] [1736290462.479650]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit[pub_text-13] killing on exit +[WARN] [1736290463.830504]: Plan exhausted +[WARN] [1736290463.868100]: Plan exhausted +[WARN] [1736290463.912185]: Plan exhausted +[WARN] [1736290463.951863]: Plan exhausted +[INFO] [1736290463.275718]: Stored episode 5. +[INFO] [1736290463.277512]: Saving video... +[INFO] [1736290463.879246]: Video saved! +[INFO] [1736290463.940228]: resetting environment +[INFO] [1736290463.966439]: cleared current plan +[INFO] [1736290463.975750]: moving to home +[INFO] [1736290467.686070]: reached home +[INFO] [1736290467.689941]: reset action history +[INFO] [1736290470.519409]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/files/requirements.txt b/em58uo70/wandb/run-20250107_175046-em58uo70/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-metadata.json b/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0f219319662a03ab899905e5123e808acd6f5db2 --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T22:50:46.621332", + "startedAt": "2025-01-07T22:50:46.206437", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.8066249999997, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.565, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.989, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 783.1284484863281 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json b/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c357dff35641ca9e61bb0821434018fa2288e256 --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "424dad725925d47367ef4841880d62f929c2f12cbb49df4ab8eb6cc05831374e", "size": 1138064, "path": "media/videos/eval/0_eval_0_424dad725925d47367ef.mp4"}, "global_step": 665, "_timestamp": 1736290475.0230422, "_runtime": 228.80320119857788, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "5b5bcc7391582f2bd11469686e27dd185745aaee20a4695c4fb61f09c3bed51b", "size": 1136231, "path": "media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4"}, "eval/num_success": 5.0, "episode": 50.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "2809d134000f06b2ecbc2350c5a4781efd46746d2d793b6d50edfe08b1c6047c", "size": 1138373, "path": "media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "8e5f95db56c8842b2074898009377c6f5b1e39e7116f9e62e5353df63532f0e1", "size": 1133816, "path": "media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "2b0616a340d3be9a0adda599d1c3d2320f10eead7ccf080832ba674ef124cedc", "size": 1196708, "path": "media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://cy4r8nazf4n7eytoo6q0tiz530yfvvlm8hn22kaek40zlxlhajglonmfnczc5etbtz6t7ycll2frq1onwtqhiiu2sxied3vsfhkkielru559knwo18q2c2i26w1gi7m9/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://q7e8b3x3aftgkrqjgi6ltzbx7hjbiuam5c0om18vp4t38rxx2b7dpsq6jph0oqk0uhotzow3he01y6mxarwn7oxgckbm8q77ftr6hzj289a0ztflqdtf6jaq9jf14jau:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 50, "total_num_episodes": 50, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9978451429801887, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.9350126980865203, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.9438994674986911, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "c48addb482dcbe6086b7b5461b7da620323bd1802b73eed00cdeaae58481ad98", "size": 420, "artifact_path": "wandb-client-artifact://rce9nh3mn2qnpayrjpqqmboxz7b4qr0297uyblpm5e5ahbq5skhhmbs3v31h84lxomipsimpugxwyoabicbtl57i287s3t2gz3yq0i4k7p4aujwt32bnow8i88udv3uy/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://zzz0iu003rgarz1ckuiaihe8w3bhxqtl2z3cmisyxrjqdsor276wjjnl4816lkzrc0569g8i0mto6ipuj15sadzcln50x17z8d3k2ffcwscski262ndj72vgdaaiqm9l:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 234}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9978451132774353, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.935012698173523, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.9438994526863098, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/logs/debug-internal.log b/em58uo70/wandb/run-20250107_175046-em58uo70/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..47e85cf593432f19866e67386535287ba26a516a --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/logs/debug-internal.log @@ -0,0 +1,727 @@ +2025-01-07 17:50:46,219 INFO StreamThr :2204041 [internal.py:wandb_internal():85] W&B internal server running at pid: 2204041, started at: 2025-01-07 17:50:46.219571 +2025-01-07 17:50:46,220 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:50:46,232 INFO WriterThread:2204041 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/run-em58uo70.wandb +2025-01-07 17:50:46,232 DEBUG SenderThread:2204041 [sender.py:send():379] send: header +2025-01-07 17:50:46,245 DEBUG SenderThread:2204041 [sender.py:send():379] send: run +2025-01-07 17:50:46,510 INFO SenderThread:2204041 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files +2025-01-07 17:50:46,510 INFO SenderThread:2204041 [sender.py:_start_run_threads():1188] run started: em58uo70 with start time 1736290246.219841 +2025-01-07 17:50:46,527 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:50:46,527 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:50:46,603 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:50:46,608 DEBUG HandlerThread:2204041 [system_info.py:__init__():26] System info init +2025-01-07 17:50:46,608 DEBUG HandlerThread:2204041 [system_info.py:__init__():41] System info init done +2025-01-07 17:50:46,608 INFO HandlerThread:2204041 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:50:46,608 INFO SystemMonitor:2204041 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:50:46,608 INFO HandlerThread:2204041 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:50:46,609 INFO SystemMonitor:2204041 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:50:46,609 INFO SystemMonitor:2204041 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:50:46,610 INFO SystemMonitor:2204041 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:50:46,611 INFO SystemMonitor:2204041 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:50:46,611 INFO SystemMonitor:2204041 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:50:46,621 DEBUG HandlerThread:2204041 [system_info.py:probe():152] Probing system +2025-01-07 17:50:46,622 DEBUG HandlerThread:2204041 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:50:46,627 DEBUG HandlerThread:2204041 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:50:46,627 DEBUG HandlerThread:2204041 [system_info.py:probe():200] Probing system done +2025-01-07 17:50:46,627 DEBUG HandlerThread:2204041 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:50:46.621332', 'startedAt': '2025-01-07T22:50:46.206437', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.8066249999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.565, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.989, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 783.1284484863281}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:50:46,627 INFO HandlerThread:2204041 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:50:46,627 INFO HandlerThread:2204041 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:50:46,627 DEBUG HandlerThread:2204041 [system_info.py:_save_code():44] Saving code +2025-01-07 17:50:46,633 DEBUG HandlerThread:2204041 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:50:46,633 DEBUG HandlerThread:2204041 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:50:46,700 DEBUG HandlerThread:2204041 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:50:46,700 INFO HandlerThread:2204041 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:50:46,704 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:50:46,704 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:50:46,704 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:50:46,704 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:50:46,945 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/lchhyf4v-wandb-metadata.json +2025-01-07 17:50:46,958 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:50:46,959 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:50:46,959 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:50:46,959 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:50:46,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:46,962 INFO HandlerThread:2204041 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/tb" + save: true +} +control { + relay_id: "9d3497cdfa604630b5ad827e26575f59" +} +_info { + stream_id: "em58uo70" +} + +2025-01-07 17:50:46,962 DEBUG HandlerThread:2204041 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:50:47,001 INFO wandb-upload_2:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/eeyox32h-diff.patch +2025-01-07 17:50:47,011 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/2htrlx2o-code/FISH/eval_robot.py +2025-01-07 17:50:47,070 DEBUG SenderThread:2204041 [sender.py:send():379] send: telemetry +2025-01-07 17:50:47,070 DEBUG SenderThread:2204041 [sender.py:send():379] send: tbrecord +2025-01-07 17:50:47,071 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:50:47,071 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 with policy live +2025-01-07 17:50:47,266 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/kt5i0mta-events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:50:47,512 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/code/FISH/eval_robot.py +2025-01-07 17:50:47,512 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/requirements.txt +2025-01-07 17:50:47,512 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:50:47,512 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-metadata.json +2025-01-07 17:50:47,512 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:50:47,512 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/diff.patch +2025-01-07 17:50:47,513 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/code +2025-01-07 17:50:47,513 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/code/FISH +2025-01-07 17:50:47,959 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:48,959 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:49,511 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:50:49,959 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:50,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:51,303 DEBUG SenderThread:2204041 [sender.py:send():379] send: config +2025-01-07 17:50:51,304 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:50:51,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:52,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:53,512 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:50:53,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:54,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:55,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:56,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:50:56,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:57,513 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:50:57,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:58,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:59,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:00,977 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:01,514 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:01,959 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:51:01,959 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:51:02,005 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:02,038 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:02,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:03,515 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:03,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:04,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:05,516 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:05,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:06,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:07,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:07,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:08,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:09,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:10,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:11,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:12,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:12,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:13,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:14,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:15,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:16,959 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:51:16,959 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:51:17,005 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:17,963 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:18,092 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:18,519 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/config.yaml +2025-01-07 17:51:18,963 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:19,963 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:20,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:21,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:22,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:23,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:23,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:24,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:25,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:26,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:27,521 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:27,964 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:28,965 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:29,089 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:29,522 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:29,965 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:30,965 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:31,523 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:31,576 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:51:31,576 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 with policy now +2025-01-07 17:51:31,576 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:51:31,577 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:51:31,577 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:51:31,577 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:51:31,839 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/6tp6y3yi-media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 +2025-01-07 17:51:31,959 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:51:31,959 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:51:31,965 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:32,523 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:51:32,524 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 +2025-01-07 17:51:32,524 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval +2025-01-07 17:51:32,525 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos +2025-01-07 17:51:32,525 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media +2025-01-07 17:51:32,830 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/zs22w709-media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 +2025-01-07 17:51:32,965 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:33,524 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:33,966 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:34,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:35,055 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:35,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:36,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:37,525 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:37,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:38,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:39,526 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:39,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:40,599 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:40,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:41,527 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:51:41,527 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:41,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:42,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:43,527 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:43,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:44,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:45,528 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:45,881 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:45,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:46,612 DEBUG SystemMonitor:2204041 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:51:46,663 DEBUG SenderThread:2204041 [sender.py:send():379] send: stats +2025-01-07 17:51:46,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:51:46,960 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:51:47,005 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:47,529 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:47,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:48,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:49,530 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:51:49,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:50,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:51,525 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:51,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:52,968 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:53,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:54,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:55,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:56,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:51:56,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:57,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:58,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:51:59,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:00,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:01,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:52:01,960 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:52:02,001 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:02,047 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:02,970 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:03,970 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:04,970 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:05,970 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:06,970 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:07,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:07,971 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:08,971 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:09,971 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:10,971 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:11,536 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:11,971 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:12,773 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:52:12,773 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 with policy now +2025-01-07 17:52:12,776 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:52:12,776 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:52:12,776 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:52:12,777 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:52:12,971 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:13,200 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/4n6po6lp-media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 +2025-01-07 17:52:13,537 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:13,537 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:52:13,537 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 +2025-01-07 17:52:13,538 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval +2025-01-07 17:52:13,802 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:13,828 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/na48ggjt-media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 +2025-01-07 17:52:13,972 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:14,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:15,538 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:15,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:16,613 DEBUG SenderThread:2204041 [sender.py:send():379] send: stats +2025-01-07 17:52:16,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:52:16,962 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:52:16,977 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:17,539 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:17,979 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:18,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:19,119 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:19,979 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:20,972 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:21,540 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:21,972 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:22,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:23,348 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:52:23,349 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:52:23,349 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:52:23,541 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:52:23,541 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:52:23,806 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/f8ukn2cw-events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:52:23,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:24,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:25,101 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:25,541 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:25,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:26,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:27,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:28,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:29,543 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:29,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:30,175 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:30,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:31,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:52:31,961 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:52:32,001 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:32,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:33,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:34,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:35,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:36,057 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:36,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:37,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:38,974 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:39,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:40,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:41,967 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:41,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:42,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:43,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:44,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:45,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:46,613 DEBUG SenderThread:2204041 [sender.py:send():379] send: stats +2025-01-07 17:52:46,960 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:52:46,961 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:52:47,001 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:47,090 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:47,976 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:48,976 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:49,977 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:50,977 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:51,978 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:52,969 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:52,978 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:53,978 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:54,979 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:55,081 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:52:55,081 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:52:55,082 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 with policy now +2025-01-07 17:52:55,082 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:52:55,082 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:52:55,083 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:52:55,351 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/da1jldbk-media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 +2025-01-07 17:52:55,551 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:55,552 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:52:55,552 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 +2025-01-07 17:52:55,552 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval +2025-01-07 17:52:55,828 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/33c2ull0-media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 +2025-01-07 17:52:55,985 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:56,985 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:57,552 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:52:57,973 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:52:57,989 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:58,982 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:52:59,987 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:00,985 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:01,553 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:01,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:53:01,961 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:53:01,984 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:02,993 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:03,119 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:03,554 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:03,986 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:04,989 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:05,183 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:53:05,183 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:53:05,184 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:53:05,555 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:53:05,555 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:05,555 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:53:05,989 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:06,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:07,555 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:07,988 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:08,700 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:08,990 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:09,556 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:09,996 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:10,996 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:11,557 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:11,996 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:12,996 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:13,970 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:13,996 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:14,996 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:15,996 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:16,614 DEBUG SenderThread:2204041 [sender.py:send():379] send: stats +2025-01-07 17:53:16,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:53:16,961 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:53:17,001 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:17,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:18,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:19,038 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:19,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:20,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:21,560 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:21,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:22,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:23,997 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:24,970 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:24,998 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:25,998 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:26,998 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:27,998 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:28,998 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:29,971 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:29,999 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:30,999 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:31,961 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:53:31,961 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:53:32,005 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:32,999 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:33,999 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:34,999 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:35,100 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:36,003 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:36,981 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:53:36,981 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 with policy now +2025-01-07 17:53:36,984 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:53:36,984 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:53:36,984 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:53:36,985 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:53:37,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:37,251 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/699weixh-media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 +2025-01-07 17:53:37,566 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:37,566 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:53:37,566 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 +2025-01-07 17:53:37,566 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval +2025-01-07 17:53:37,850 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/dna5w348-media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 +2025-01-07 17:53:38,006 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:39,011 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:39,566 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:40,006 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:40,727 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:41,027 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:41,567 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:42,019 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:43,014 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:44,009 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:45,018 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:45,568 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:46,020 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:46,614 DEBUG SenderThread:2204041 [sender.py:send():379] send: stats +2025-01-07 17:53:46,614 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:46,963 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:53:46,964 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:53:47,021 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:48,020 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:49,022 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:50,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:50,239 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:53:50,239 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:53:50,239 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:53:50,570 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:53:50,570 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:53:50,867 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/x5t4ko3i-events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:53:51,022 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:51,570 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:51,979 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:52,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:53,021 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:53,571 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:54,027 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:55,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:55,572 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:56,023 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:57,023 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:57,573 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:53:57,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:53:58,023 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:53:59,023 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:00,023 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:01,023 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:01,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:54:01,962 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:54:02,024 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:03,024 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:03,082 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:04,024 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:05,024 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:06,024 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:07,024 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:08,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:08,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:09,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:10,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:11,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:12,025 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:13,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:13,975 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:14,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:15,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:16,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:16,615 DEBUG SenderThread:2204041 [sender.py:send():379] send: stats +2025-01-07 17:54:16,962 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:54:16,962 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:54:17,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:18,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:19,026 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:19,101 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:20,027 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:21,027 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:22,027 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:23,031 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:23,582 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:23,927 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:54:23,927 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 with policy now +2025-01-07 17:54:23,939 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:54:23,939 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:54:23,940 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:23,940 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:24,033 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:24,186 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/rb8yjj6w-media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 +2025-01-07 17:54:24,582 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:54:24,582 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 +2025-01-07 17:54:24,582 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval +2025-01-07 17:54:24,913 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/7f41e2bk-media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 +2025-01-07 17:54:24,977 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:25,037 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:25,582 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:26,040 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:27,039 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:28,041 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:29,044 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:29,584 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:30,050 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:30,520 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:31,074 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:31,584 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:31,966 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:54:31,967 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:54:32,054 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:33,066 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:34,055 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:34,308 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:54:34,309 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:34,310 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:34,541 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:54:34,542 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:54:34,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:54:34,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:54:34,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:54:34,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media +2025-01-07 17:54:34,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval +2025-01-07 17:54:34,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table +2025-01-07 17:54:34,727 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:54:34,784 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/wspzzxuq-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:54:35,031 INFO wandb-upload_0:2204041 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyp8pg8x3 +2025-01-07 17:54:35,055 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:35,113 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:54:35,157 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:54:35,453 INFO SenderThread:2204041 [sender.py:send_request_log_artifact():1525] logged artifact run-em58uo70-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjAxNzgzNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcyNzQ3MQ==', 'latestArtifact': None}} +2025-01-07 17:54:35,453 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:54:35,453 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-07 17:54:35,453 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:54:35,453 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,454 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,454 DEBUG SenderThread:2204041 [sender.py:send():379] send: telemetry +2025-01-07 17:54:35,454 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,458 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,458 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,458 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,458 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,459 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,459 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,459 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,459 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,459 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,460 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,460 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,460 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,460 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,461 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,461 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,461 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,461 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,462 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,462 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,462 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:35,462 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:35,463 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:54:35,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:54:35,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:35,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:54:35,586 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json +2025-01-07 17:54:35,587 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval +2025-01-07 17:54:35,719 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/kvw6i9vs-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:54:35,774 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/uax9iuap-media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json +2025-01-07 17:54:35,845 INFO wandb-upload_2:2204041 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpx_qywsap +2025-01-07 17:54:36,055 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:36,240 INFO SenderThread:2204041 [sender.py:send_request_log_artifact():1525] logged artifact run-em58uo70-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjAxNzg2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcyNzQ3Ng==', 'latestArtifact': None}} +2025-01-07 17:54:36,245 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:36,387 DEBUG SenderThread:2204041 [sender.py:send():379] send: files +2025-01-07 17:54:36,387 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json with policy now +2025-01-07 17:54:36,388 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:54:36,388 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:36,388 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:36,587 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/config.yaml +2025-01-07 17:54:36,587 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:54:36,655 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/c4mmi5oo-media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json +2025-01-07 17:54:37,055 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:37,587 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:38,055 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:39,056 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:40,056 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:40,925 DEBUG SenderThread:2204041 [sender.py:send():379] send: exit +2025-01-07 17:54:40,926 INFO SenderThread:2204041 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:54:40,926 INFO SenderThread:2204041 [sender.py:send_exit():588] handling runtime: 234 +2025-01-07 17:54:40,926 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:40,926 INFO SenderThread:2204041 [sender.py:send_exit():594] send defer +2025-01-07 17:54:40,926 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:40,926 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:54:40,926 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:40,927 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:54:40,927 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:54:40,927 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:40,927 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:54:40,927 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:40,927 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:54:40,927 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:54:40,927 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:40,927 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:54:40,927 INFO HandlerThread:2204041 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:54:40,927 DEBUG SystemMonitor:2204041 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:54:40,927 DEBUG SystemMonitor:2204041 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:54:40,928 INFO HandlerThread:2204041 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:54:40,928 INFO HandlerThread:2204041 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:54:40,997 INFO HandlerThread:2204041 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:54:40,997 INFO HandlerThread:2204041 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:54:40,997 INFO HandlerThread:2204041 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:54:40,998 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:40,998 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:54:40,998 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:54:40,998 DEBUG SenderThread:2204041 [sender.py:send():379] send: stats +2025-01-07 17:54:40,998 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:40,998 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:54:40,999 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:40,999 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:54:40,999 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:54:40,999 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:40,999 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:54:41,589 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:54:43,590 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:47,315 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:54:47,315 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:47,315 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:54:47,315 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:47,317 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:47,317 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:54:47,317 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:54:47,317 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:54:47,317 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:47,317 DEBUG SenderThread:2204041 [sender.py:send():379] send: history +2025-01-07 17:54:47,317 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:54:47,317 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:54:47,319 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:47,319 DEBUG SenderThread:2204041 [sender.py:send():379] send: summary +2025-01-07 17:54:47,323 INFO SenderThread:2204041 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:54:47,323 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:47,323 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:54:47,323 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:54:47,323 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:47,323 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:54:47,324 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:47,324 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:54:47,324 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:54:47,324 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:54:47,324 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:47,324 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:54:47,324 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:47,324 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:54:47,591 INFO Thread-12 :2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:54:47,927 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:54:50,723 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:54:50,723 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:54:50,724 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:50,724 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:54:50,724 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:50,724 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:54:50,724 INFO SenderThread:2204041 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:54:50,726 INFO SenderThread:2204041 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:54:50,728 INFO SenderThread:2204041 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:54:50,742 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:54:50,743 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:50,743 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:54:50,743 DEBUG SenderThread:2204041 [sender.py:send():379] send: artifact +2025-01-07 17:54:50,928 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:54:50,935 INFO SenderThread:2204041 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 17:54:50,936 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:50,936 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:54:50,936 INFO SenderThread:2204041 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:54:51,592 INFO SenderThread:2204041 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:51,593 INFO SenderThread:2204041 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files +2025-01-07 17:54:51,593 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/config.yaml config.yaml +2025-01-07 17:54:51,593 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json wandb-summary.json +2025-01-07 17:54:51,593 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/requirements.txt requirements.txt +2025-01-07 17:54:51,593 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:54:51,593 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:54:51,594 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log output.log +2025-01-07 17:54:51,594 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/diff.patch diff.patch +2025-01-07 17:54:51,598 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:54:51,599 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 media/videos/eval/2_eval_2_2809d134000f06b2ecbc.mp4 +2025-01-07 17:54:51,599 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 media/videos/eval/4_eval_4_2b0616a340d3be9a0add.mp4 +2025-01-07 17:54:51,599 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 media/videos/eval/1_eval_1_5b5bcc7391582f2bd114.mp4 +2025-01-07 17:54:51,599 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 media/videos/eval/3_eval_3_8e5f95db56c8842b2074.mp4 +2025-01-07 17:54:51,599 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 media/videos/eval/0_eval_0_424dad725925d47367ef.mp4 +2025-01-07 17:54:51,601 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:54:51,601 INFO SenderThread:2204041 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json media/table/eval/total_success_rate_ci_6_c48addb482dcbe6086b7.table.json +2025-01-07 17:54:51,601 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:54:51,602 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:54:51,603 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:51,603 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:54:51,603 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:51,603 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:54:51,603 INFO SenderThread:2204041 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:54:51,786 INFO wandb-upload_2:2204041 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/config.yaml +2025-01-07 17:54:51,795 INFO wandb-upload_1:2204041 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/requirements.txt +2025-01-07 17:54:51,820 INFO wandb-upload_0:2204041 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/wandb-summary.json +2025-01-07 17:54:51,823 INFO wandb-upload_4:2204041 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/files/output.log +2025-01-07 17:54:51,906 INFO wandb-upload_3:2204041 [upload_job.py:push():130] Uploaded file /tmp/tmplfm56fb5wandb/v587226h-events.out.tfevents.1736290246.leonmkim-ROG-Strix-G15CS-G15CS.2203862.0 +2025-01-07 17:54:51,928 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:54:51,928 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:54:52,106 INFO Thread-11 (_thread_body):2204041 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:54:52,106 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:52,106 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:54:52,106 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:52,106 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:54:52,107 INFO SenderThread:2204041 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:54:52,108 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:54:52,108 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:52,108 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:54:52,108 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:52,108 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:54:52,108 INFO SenderThread:2204041 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:54:52,305 INFO SenderThread:2204041 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:54:52,305 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:54:52,305 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:52,305 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:54:52,305 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:52,305 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:54:52,306 INFO SenderThread:2204041 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:54:52,306 DEBUG SenderThread:2204041 [sender.py:send():379] send: final +2025-01-07 17:54:52,306 DEBUG SenderThread:2204041 [sender.py:send():379] send: footer +2025-01-07 17:54:52,306 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:54:52,306 INFO HandlerThread:2204041 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:54:52,306 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: defer +2025-01-07 17:54:52,306 INFO SenderThread:2204041 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:54:52,307 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:54:52,307 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:54:52,307 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:54:52,307 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:54:52,309 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:54:52,309 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:54:52,310 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:54:52,311 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:54:52,364 DEBUG SenderThread:2204041 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:54:52,365 INFO MainThread:2204041 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:54:52,365 INFO MainThread:2204041 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:54:52,365 INFO MainThread:2204041 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:54:52,366 DEBUG HandlerThread:2204041 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:54:52,366 INFO HandlerThread:2204041 [handler.py:finish():882] shutting down handler +2025-01-07 17:54:53,309 INFO WriterThread:2204041 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/run-em58uo70.wandb +2025-01-07 17:54:53,364 INFO SenderThread:2204041 [sender.py:finish():1615] shutting down sender +2025-01-07 17:54:53,365 INFO SenderThread:2204041 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:54:53,365 INFO SenderThread:2204041 [file_pusher.py:join():175] waiting for file pusher diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/logs/debug.log b/em58uo70/wandb/run-20250107_175046-em58uo70/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e54204a772cd325e62e0dd0a9b8697906bc0e040 --- /dev/null +++ b/em58uo70/wandb/run-20250107_175046-em58uo70/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Configure stats pid to 2203862 +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/settings +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/logs/debug.log +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/wandb/run-20250107_175046-em58uo70/logs/debug-internal.log +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70'} +2025-01-07 17:50:46,213 INFO MainThread:2203862 [wandb_init.py:init():619] starting backend +2025-01-07 17:50:46,214 INFO MainThread:2203862 [wandb_init.py:init():623] setting up manager +2025-01-07 17:50:46,218 INFO MainThread:2203862 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:50:46,219 INFO MainThread:2203862 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:50:46,231 INFO MainThread:2203862 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:50:46,241 INFO MainThread:2203862 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:50:46,526 INFO MainThread:2203862 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:50:46,596 INFO MainThread:2203862 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:50:46,597 INFO MainThread:2203862 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:50:46,959 INFO MainThread:2203862 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:50:46,959 INFO MainThread:2203862 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:50:46,959 INFO MainThread:2203862 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:50:46,959 INFO MainThread:2203862 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:50:46,961 INFO MainThread:2203862 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:50:46,961 INFO MainThread:2203862 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/em58uo70/tb, True +2025-01-07 17:50:51,303 INFO MainThread:2203862 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 17:54:53,366 WARNING MsgRouterThr:2203862 [router.py:message_loop():77] message_loop has been closed diff --git a/em58uo70/wandb/run-20250107_175046-em58uo70/run-em58uo70.wandb b/em58uo70/wandb/run-20250107_175046-em58uo70/run-em58uo70.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2bdc05d613d8b7a6d29a18f131620ffebd063e5c Binary files /dev/null and b/em58uo70/wandb/run-20250107_175046-em58uo70/run-em58uo70.wandb differ