diff --git a/.gitattributes b/.gitattributes index 539dd1c0cca87574b556e5ba296f7c1aaf306525..7c556d77a10f108517225a5b22fd552bcb4bea53 100644 --- a/.gitattributes +++ b/.gitattributes @@ -113,3 +113,24 @@ aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/2_eval_2_96f aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 filter=lfs diff=lfs merge=lfs -text aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 filter=lfs diff=lfs merge=lfs -text aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 filter=lfs diff=lfs merge=lfs -text +aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 filter=lfs diff=lfs merge=lfs -text +aqyye28j/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +aqyye28j/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +aqyye28j/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +aqyye28j/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +aqyye28j/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +m9j39xya/episode_rosbags/episode_0_2025-01-21-22-13-17.bag filter=lfs diff=lfs merge=lfs -text +m9j39xya/episode_rosbags/episode_1_2025-01-21-22-13-43.bag filter=lfs diff=lfs merge=lfs -text +m9j39xya/episode_rosbags/episode_2_2025-01-21-22-14-12.bag filter=lfs diff=lfs merge=lfs -text +07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 filter=lfs diff=lfs merge=lfs -text +07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 filter=lfs diff=lfs merge=lfs -text +aqyye28j/episode_rosbags/episode_1_2025-01-21-22-08-24.bag filter=lfs diff=lfs merge=lfs -text +07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 filter=lfs diff=lfs merge=lfs -text +07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 filter=lfs diff=lfs merge=lfs -text +07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 filter=lfs diff=lfs merge=lfs -text +aqyye28j/episode_rosbags/episode_2_2025-01-21-22-08-51.bag filter=lfs diff=lfs merge=lfs -text +07nra58y/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +07nra58y/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +07nra58y/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +07nra58y/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +07nra58y/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/07nra58y/.hydra/config.yaml b/07nra58y/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dc75d61dbe4b7ba0ede1ab77029649cc5b18502 --- /dev/null +++ b/07nra58y/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/07nra58y/.hydra/hydra.yaml b/07nra58y/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b1a7d839eacd6bd57a89f97a2941f50577e07d7 --- /dev/null +++ b/07nra58y/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/07nra58y/.hydra/overrides.yaml b/07nra58y/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/07nra58y/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/07nra58y/episode_rosbags/aligned_depth_to_color_K.npy b/07nra58y/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/07nra58y/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/07nra58y/episode_rosbags/cam_tf_world.npy b/07nra58y/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/07nra58y/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/07nra58y/episode_rosbags/color_K.npy b/07nra58y/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/07nra58y/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/07nra58y/episode_rosbags/depth_K.npy b/07nra58y/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/07nra58y/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/07nra58y/eval_robot.log b/07nra58y/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..24e1a87be99d248842625c3377254b0f79c3119d --- /dev/null +++ b/07nra58y/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 20:41:54,496][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 20:41:54,498][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/07nra58y/eval_video/0_eval.mp4 b/07nra58y/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..88683a0a01cd11a49f9b34beaa830e09f7d03a41 --- /dev/null +++ b/07nra58y/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4947144f835c80690116d2cb0e8332f4d2ceca6ec5bfb8c58a4b4f20ee84417 +size 633108 diff --git a/07nra58y/eval_video/1_eval.mp4 b/07nra58y/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5bf51f3163afee71ce26507fa8ae2034832f63d3 --- /dev/null +++ b/07nra58y/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d488251d8ba1dc3b3f72748124f7dbac02cf3572014e65f6ea0d79d795b00547 +size 675044 diff --git a/07nra58y/eval_video/2_eval.mp4 b/07nra58y/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8e55762771773641ca1779b8034ef7c91ef5f4d9 --- /dev/null +++ b/07nra58y/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c909f78c2729a4b3bd82117634333baf6c377e4bff3bedb8f26f8c7b7aae8a97 +size 677702 diff --git a/07nra58y/eval_video/3_eval.mp4 b/07nra58y/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0296611c4966d387f745d6b350feaeb727b6c73b --- /dev/null +++ b/07nra58y/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f946236f6700e74eb861ec8865fe687033f8099ed566b6d47b52734366fe6f7 +size 786909 diff --git a/07nra58y/eval_video/4_eval.mp4 b/07nra58y/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b4dc940a829fc4523e8ccefc79fb383106ee7b26 --- /dev/null +++ b/07nra58y/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd3afb63cd85bd424ac4f2182808e18600f37d681ebff8a80d3dd694c1c98823 +size 726949 diff --git a/07nra58y/tb/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 b/07nra58y/tb/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 new file mode 100644 index 0000000000000000000000000000000000000000..504f058eed2dec8d0e262937fb8d7d6529d52580 --- /dev/null +++ b/07nra58y/tb/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bec0f11d39fdd14c2a2bcd6899c8399cbbe01dfebbbe97c71d4bf7ba8f8c8aef +size 6499 diff --git a/07nra58y/wandb/debug-internal.log b/07nra58y/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f78741371e57d026be6c451821690b7f9789047a --- /dev/null +++ b/07nra58y/wandb/debug-internal.log @@ -0,0 +1,711 @@ +2025-01-21 20:42:00,346 INFO StreamThr :2639291 [internal.py:wandb_internal():85] W&B internal server running at pid: 2639291, started at: 2025-01-21 20:42:00.345235 +2025-01-21 20:42:00,347 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status +2025-01-21 20:42:00,357 INFO WriterThread:2639291 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/run-07nra58y.wandb +2025-01-21 20:42:00,358 DEBUG SenderThread:2639291 [sender.py:send():379] send: header +2025-01-21 20:42:00,366 DEBUG SenderThread:2639291 [sender.py:send():379] send: run +2025-01-21 20:42:00,672 INFO SenderThread:2639291 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files +2025-01-21 20:42:00,672 INFO SenderThread:2639291 [sender.py:_start_run_threads():1188] run started: 07nra58y with start time 1737510120.345791 +2025-01-21 20:42:00,689 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 20:42:00,689 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: check_version +2025-01-21 20:42:00,763 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 20:42:00,768 DEBUG HandlerThread:2639291 [system_info.py:__init__():26] System info init +2025-01-21 20:42:00,768 DEBUG HandlerThread:2639291 [system_info.py:__init__():41] System info init done +2025-01-21 20:42:00,768 INFO HandlerThread:2639291 [system_monitor.py:start():194] Starting system monitor +2025-01-21 20:42:00,768 INFO SystemMonitor:2639291 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 20:42:00,768 INFO HandlerThread:2639291 [system_monitor.py:probe():214] Collecting system info +2025-01-21 20:42:00,769 INFO SystemMonitor:2639291 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 20:42:00,769 INFO SystemMonitor:2639291 [interfaces.py:start():188] Started disk monitoring +2025-01-21 20:42:00,770 INFO SystemMonitor:2639291 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 20:42:00,771 INFO SystemMonitor:2639291 [interfaces.py:start():188] Started memory monitoring +2025-01-21 20:42:00,771 INFO SystemMonitor:2639291 [interfaces.py:start():188] Started network monitoring +2025-01-21 20:42:00,781 DEBUG HandlerThread:2639291 [system_info.py:probe():152] Probing system +2025-01-21 20:42:00,783 DEBUG HandlerThread:2639291 [system_info.py:_probe_git():137] Probing git +2025-01-21 20:42:00,788 DEBUG HandlerThread:2639291 [system_info.py:_probe_git():145] Probing git done +2025-01-21 20:42:00,788 DEBUG HandlerThread:2639291 [system_info.py:probe():200] Probing system done +2025-01-21 20:42:00,788 DEBUG HandlerThread:2639291 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T01:42:00.781581', 'startedAt': '2025-01-22T01:42:00.329941', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.0602499999995, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4495.897, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.985, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 311.03546142578125}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 20:42:00,788 INFO HandlerThread:2639291 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 20:42:00,788 INFO HandlerThread:2639291 [system_monitor.py:probe():227] Publishing system info +2025-01-21 20:42:00,788 DEBUG HandlerThread:2639291 [system_info.py:_save_code():44] Saving code +2025-01-21 20:42:00,794 DEBUG HandlerThread:2639291 [system_info.py:_save_code():67] Saving code done +2025-01-21 20:42:00,794 DEBUG HandlerThread:2639291 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 20:42:00,856 DEBUG HandlerThread:2639291 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 20:42:00,857 INFO HandlerThread:2639291 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 20:42:00,860 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:42:00,860 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 20:42:00,860 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 20:42:00,860 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 20:42:01,102 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 20:42:01,102 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: python_packages +2025-01-21 20:42:01,102 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:42:01,102 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:42:01,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:01,134 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/6znpn0sz-wandb-metadata.json +2025-01-21 20:42:01,145 INFO wandb-upload_2:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/z0y8zo0r-diff.patch +2025-01-21 20:42:01,145 INFO HandlerThread:2639291 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/tb" + save: true +} +control { + relay_id: "e16b6b475cee4a50a1ed47d57c820f7d" +} +_info { + stream_id: "07nra58y" +} + +2025-01-21 20:42:01,146 DEBUG HandlerThread:2639291 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 20:42:01,186 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/s878f3g8-code/FISH/eval_robot.py +2025-01-21 20:42:01,268 DEBUG SenderThread:2639291 [sender.py:send():379] send: telemetry +2025-01-21 20:42:01,268 DEBUG SenderThread:2639291 [sender.py:send():379] send: tbrecord +2025-01-21 20:42:01,269 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:42:01,269 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 with policy live +2025-01-21 20:42:01,469 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/js8hmcxv-events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:42:01,673 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-metadata.json +2025-01-21 20:42:01,673 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:42:01,674 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/code/FISH/eval_robot.py +2025-01-21 20:42:01,674 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:01,674 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/requirements.txt +2025-01-21 20:42:01,674 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/diff.patch +2025-01-21 20:42:01,674 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/code +2025-01-21 20:42:01,674 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/code/FISH +2025-01-21 20:42:02,102 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:03,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:03,673 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:04,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:05,088 DEBUG SenderThread:2639291 [sender.py:send():379] send: config +2025-01-21 20:42:05,129 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:06,102 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:06,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:07,141 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:07,675 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:08,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:09,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:09,675 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:10,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:11,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:11,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:11,676 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:12,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:13,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:13,676 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:14,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:15,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:15,677 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:16,102 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:42:16,102 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:42:16,145 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:16,230 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:17,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:18,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:19,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:20,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:21,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:22,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:22,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:23,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:24,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:25,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:26,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:27,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:27,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:28,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:29,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:30,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:31,102 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:42:31,102 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:42:31,145 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:32,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:32,224 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:32,681 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/config.yaml +2025-01-21 20:42:33,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:34,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:35,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:36,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:37,107 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:38,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:38,107 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:39,107 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:40,107 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:41,107 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:42,107 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:43,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:43,107 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:44,108 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:45,108 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:46,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:42:46,103 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:42:46,145 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:47,108 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:48,108 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:48,185 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:49,108 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:50,108 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:51,108 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:52,109 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:53,109 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:53,687 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:53,905 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:54,109 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:55,109 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:55,687 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:56,110 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:57,110 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:57,688 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:58,110 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:59,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:59,110 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:00,112 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:00,772 DEBUG SystemMonitor:2639291 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 20:43:00,772 DEBUG SenderThread:2639291 [sender.py:send():379] send: stats +2025-01-21 20:43:01,102 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:43:01,103 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:43:01,145 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:02,112 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:03,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:04,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:04,170 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:05,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:06,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:07,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:08,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:09,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:09,667 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:09,691 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:09,900 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:43:09,900 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 with policy now +2025-01-21 20:43:09,900 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:43:09,901 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:43:09,901 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:43:09,902 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:43:10,114 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/pw6i8cez-media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 +2025-01-21 20:43:10,116 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:10,692 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:43:10,692 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 +2025-01-21 20:43:10,692 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos +2025-01-21 20:43:10,693 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval +2025-01-21 20:43:10,693 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media +2025-01-21 20:43:10,963 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/nf36eax3-media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 +2025-01-21 20:43:11,119 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:11,692 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:12,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:13,114 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:14,114 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:15,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:15,257 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:15,693 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:16,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:43:16,103 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:43:16,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:17,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:17,694 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:18,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:19,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:20,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:21,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:21,116 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:22,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:23,116 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:24,116 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:25,116 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:26,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:26,116 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:27,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:28,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:29,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:30,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:30,773 DEBUG SenderThread:2639291 [sender.py:send():379] send: stats +2025-01-21 20:43:31,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:43:31,103 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:43:31,145 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:31,246 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:32,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:33,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:34,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:35,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:36,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:36,751 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:37,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:37,701 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:37,701 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:43:37,971 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/als8vce5-events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:43:38,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:39,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:39,701 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:40,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:41,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:41,702 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:42,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:42,369 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:43,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:43,703 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:44,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:45,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:46,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:43:46,104 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:43:46,145 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:47,119 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:48,119 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:48,189 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:49,119 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:50,120 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:51,120 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:52,120 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:53,120 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:54,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:54,120 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:55,152 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:55,229 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:43:55,229 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:43:55,229 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 with policy now +2025-01-21 20:43:55,230 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:43:55,230 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:43:55,230 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:43:55,441 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/rga3v44b-media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 +2025-01-21 20:43:55,706 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:43:55,707 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:55,707 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 +2025-01-21 20:43:55,707 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval +2025-01-21 20:43:55,964 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/duif5rw2-media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 +2025-01-21 20:43:56,120 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:57,121 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:57,707 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:58,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:59,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:59,708 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:59,789 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:00,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:00,773 DEBUG SenderThread:2639291 [sender.py:send():379] send: stats +2025-01-21 20:44:01,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:44:01,113 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:44:01,157 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:02,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:03,124 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:03,709 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:04,124 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:05,124 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:05,570 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:06,124 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:07,124 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:07,571 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:44:07,571 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:44:07,571 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:44:07,711 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:44:07,711 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:07,712 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:44:08,124 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:09,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:09,712 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:10,124 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:10,838 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:11,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:11,713 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:12,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:13,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:13,714 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:14,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:15,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:16,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:44:16,104 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:44:16,145 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:16,259 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:17,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:18,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:19,128 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:20,128 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:21,129 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:21,716 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:22,111 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:22,129 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:23,129 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:24,129 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:25,131 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:25,326 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:44:25,326 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 with policy now +2025-01-21 20:44:25,329 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:44:25,329 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:44:25,329 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:44:25,330 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:44:25,564 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/bojhi70h-media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 +2025-01-21 20:44:25,718 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:44:25,718 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:25,718 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 +2025-01-21 20:44:25,718 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval +2025-01-21 20:44:26,016 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/8woarlbs-media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 +2025-01-21 20:44:26,132 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:27,114 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:27,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:27,718 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:28,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:29,132 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:29,719 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:30,142 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:30,774 DEBUG SenderThread:2639291 [sender.py:send():379] send: stats +2025-01-21 20:44:31,121 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:44:31,121 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:44:31,165 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:32,133 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:32,713 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:33,133 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:33,720 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:34,135 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:35,134 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:36,137 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:36,386 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:44:36,386 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:44:36,387 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:44:36,721 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:44:36,722 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:44:37,173 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:37,722 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:38,138 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:38,173 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:39,134 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:39,722 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:40,134 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:41,135 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:41,723 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:42,135 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:43,135 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:44,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:44,135 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:45,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:46,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:44:46,106 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:44:46,149 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:47,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:48,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:49,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:49,261 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:50,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:51,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:52,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:53,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:54,137 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:55,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:55,137 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:56,146 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:56,989 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:44:56,989 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 with policy now +2025-01-21 20:44:56,991 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:44:56,992 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:44:56,992 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:44:56,992 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:44:57,138 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:57,205 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/s7ru733e-media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 +2025-01-21 20:44:57,729 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:44:57,729 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:57,729 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 +2025-01-21 20:44:57,730 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval +2025-01-21 20:44:57,995 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/sgrw391v-media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 +2025-01-21 20:44:58,141 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:59,147 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:59,730 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:00,146 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:00,550 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:00,774 DEBUG SenderThread:2639291 [sender.py:send():379] send: stats +2025-01-21 20:45:01,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:45:01,114 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:45:01,154 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:01,730 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:02,150 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:03,157 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:03,731 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:45:03,984 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/osryolr7-events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:45:04,155 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:04,420 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:45:04,420 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:04,421 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:04,732 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:45:05,150 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:05,613 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:05,732 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:06,153 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:07,152 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:08,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:09,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:09,733 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:10,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:10,838 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:11,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:12,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:13,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:14,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:15,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:16,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:45:16,106 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:45:16,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:16,215 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:17,152 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:18,152 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:19,152 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:20,152 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:21,153 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:21,662 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:22,153 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:23,155 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:23,213 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:45:23,213 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:45:23,213 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 with policy now +2025-01-21 20:45:23,214 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:45:23,214 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:23,214 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:23,430 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/glti81mu-media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 +2025-01-21 20:45:23,738 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:45:23,738 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:23,738 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 +2025-01-21 20:45:23,738 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval +2025-01-21 20:45:23,999 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/dlrg1wek-media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 +2025-01-21 20:45:24,167 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:25,167 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:25,739 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:26,170 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:26,738 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:27,173 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:27,739 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:28,169 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:29,169 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:30,167 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:30,774 DEBUG SenderThread:2639291 [sender.py:send():379] send: stats +2025-01-21 20:45:31,114 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:45:31,114 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:45:31,164 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:31,426 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 20:45:31,426 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 20:45:31,606 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:45:31,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:31,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:45:31,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 20:45:31,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media +2025-01-21 20:45:31,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval +2025-01-21 20:45:31,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table +2025-01-21 20:45:31,925 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/gtcdtkfg-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 20:45:31,981 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 20:45:31,987 INFO wandb-upload_0:2639291 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp2he6lwp_ +2025-01-21 20:45:31,990 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:45:32,163 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:32,645 INFO SenderThread:2639291 [sender.py:send_request_log_artifact():1525] logged artifact run-07nra58y-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxODkzNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4NjEwMg==', 'latestArtifact': None}} +2025-01-21 20:45:32,645 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:45:32,645 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,646 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:32,646 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,646 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:45:32,646 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-21 20:45:32,647 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:45:32,647 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,647 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,647 DEBUG SenderThread:2639291 [sender.py:send():379] send: telemetry +2025-01-21 20:45:32,647 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,648 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,648 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,650 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,650 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,650 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,650 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,651 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,651 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,651 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,651 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,652 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,652 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,652 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,652 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,653 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,653 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,653 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,653 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,654 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,654 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,654 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,654 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 20:45:32,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:45:32,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:45:32,742 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-21 20:45:32,742 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval +2025-01-21 20:45:32,916 INFO wandb-upload_2:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/jtt99ao5-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 20:45:32,937 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/njtoi5mo-media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-21 20:45:33,022 INFO wandb-upload_1:2639291 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1fzi00cp +2025-01-21 20:45:33,163 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:33,429 INFO SenderThread:2639291 [sender.py:send_request_log_artifact():1525] logged artifact run-07nra58y-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxODk2NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4NjEwNg==', 'latestArtifact': None}} +2025-01-21 20:45:33,429 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:45:33,429 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json with policy now +2025-01-21 20:45:33,429 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:45:33,430 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:33,430 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:33,688 INFO wandb-upload_2:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/5yww2g8g-media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-21 20:45:33,742 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:45:33,742 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:34,163 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:35,163 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:36,163 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:37,163 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:38,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:38,163 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:39,164 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:40,164 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:41,164 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:42,164 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:43,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:43,164 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:44,165 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:44,837 DEBUG SenderThread:2639291 [sender.py:send():379] send: exit +2025-01-21 20:45:44,837 INFO SenderThread:2639291 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 20:45:44,838 INFO SenderThread:2639291 [sender.py:send_exit():588] handling runtime: 224 +2025-01-21 20:45:44,838 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:44,838 INFO SenderThread:2639291 [sender.py:send_exit():594] send defer +2025-01-21 20:45:44,838 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:44,838 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 20:45:44,838 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:44,838 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 20:45:44,838 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 1 +2025-01-21 20:45:44,839 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:44,839 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 20:45:44,839 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:44,839 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 20:45:44,839 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 2 +2025-01-21 20:45:44,839 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:44,839 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 20:45:44,839 INFO HandlerThread:2639291 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 20:45:44,839 DEBUG SystemMonitor:2639291 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 20:45:44,840 DEBUG SystemMonitor:2639291 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 20:45:44,840 INFO HandlerThread:2639291 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 20:45:44,840 INFO HandlerThread:2639291 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 20:45:44,892 INFO HandlerThread:2639291 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 20:45:44,893 INFO HandlerThread:2639291 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 20:45:44,893 INFO HandlerThread:2639291 [interfaces.py:finish():200] Joined network monitor +2025-01-21 20:45:44,893 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:44,893 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 20:45:44,893 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 3 +2025-01-21 20:45:44,893 DEBUG SenderThread:2639291 [sender.py:send():379] send: stats +2025-01-21 20:45:44,894 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:44,894 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 20:45:44,894 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:44,894 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 20:45:44,894 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 4 +2025-01-21 20:45:44,894 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:44,894 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 20:45:45,746 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:45:45,747 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:49,748 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/config.yaml +2025-01-21 20:45:51,462 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:45:51,462 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:51,462 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 20:45:51,463 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:51,463 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 20:45:51,463 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 5 +2025-01-21 20:45:51,463 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:45:51,464 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:45:51,464 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:51,464 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:51,464 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 20:45:51,465 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:51,465 DEBUG SenderThread:2639291 [sender.py:send():379] send: summary +2025-01-21 20:45:51,469 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:51,469 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:51,469 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 20:45:51,469 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 6 +2025-01-21 20:45:51,469 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:51,469 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 20:45:51,469 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:51,469 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 20:45:51,469 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 7 +2025-01-21 20:45:51,470 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:51,470 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:51,470 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 20:45:51,470 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:51,470 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 20:45:51,753 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:45:51,839 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:45:53,305 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 8 +2025-01-21 20:45:53,305 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:45:53,305 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:53,305 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 20:45:53,306 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:53,306 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 20:45:53,306 INFO SenderThread:2639291 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 20:45:53,308 INFO SenderThread:2639291 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 20:45:53,310 INFO SenderThread:2639291 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 20:45:53,324 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 9 +2025-01-21 20:45:53,324 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:53,324 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 20:45:53,325 DEBUG SenderThread:2639291 [sender.py:send():379] send: artifact +2025-01-21 20:45:53,750 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:53,771 INFO wandb-upload_0:2639291 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6xpc5ztm +2025-01-21 20:45:53,839 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:45:53,899 INFO wandb-upload_1:2639291 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1__rqxph +2025-01-21 20:45:53,914 INFO wandb-upload_2:2639291 [upload_job.py:push():88] Uploaded file /tmp/tmp1vwz0ran/wandb-job.json +2025-01-21 20:45:54,450 INFO SenderThread:2639291 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYxNDI3MQ==', 'versionIndex': 452}}} +2025-01-21 20:45:54,450 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:54,450 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 20:45:54,450 INFO SenderThread:2639291 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 20:45:54,751 INFO SenderThread:2639291 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files +2025-01-21 20:45:54,751 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/config.yaml config.yaml +2025-01-21 20:45:54,751 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json wandb-summary.json +2025-01-21 20:45:54,751 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/requirements.txt requirements.txt +2025-01-21 20:45:54,751 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-metadata.json wandb-metadata.json +2025-01-21 20:45:54,751 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:45:54,753 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log output.log +2025-01-21 20:45:54,754 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/diff.patch diff.patch +2025-01-21 20:45:54,756 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 20:45:54,756 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 +2025-01-21 20:45:54,756 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 +2025-01-21 20:45:54,757 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 +2025-01-21 20:45:54,757 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 +2025-01-21 20:45:54,757 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 +2025-01-21 20:45:54,757 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-21 20:45:54,757 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 20:45:54,757 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 10 +2025-01-21 20:45:54,757 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:45:54,759 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:54,760 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 20:45:54,761 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:54,761 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 20:45:54,761 INFO SenderThread:2639291 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 20:45:54,840 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:45:54,840 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:45:54,939 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/config.yaml +2025-01-21 20:45:54,949 INFO wandb-upload_2:2639291 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/requirements.txt +2025-01-21 20:45:54,951 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:45:55,003 INFO wandb-upload_4:2639291 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:55,059 INFO wandb-upload_3:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/lqk4vyyj-events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:45:55,259 INFO Thread-11 (_thread_body):2639291 [sender.py:transition_state():614] send defer: 11 +2025-01-21 20:45:55,259 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:55,259 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 20:45:55,260 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:55,260 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 20:45:55,260 INFO SenderThread:2639291 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 20:45:55,261 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 12 +2025-01-21 20:45:55,261 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:55,261 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 20:45:55,262 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:55,262 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 20:45:55,262 INFO SenderThread:2639291 [file_stream.py:finish():601] file stream finish called +2025-01-21 20:45:55,464 INFO SenderThread:2639291 [file_stream.py:finish():605] file stream finish is done +2025-01-21 20:45:55,464 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 13 +2025-01-21 20:45:55,464 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:55,464 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 20:45:55,464 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:55,465 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 20:45:55,465 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 14 +2025-01-21 20:45:55,465 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:55,465 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 20:45:55,465 DEBUG SenderThread:2639291 [sender.py:send():379] send: final +2025-01-21 20:45:55,465 DEBUG SenderThread:2639291 [sender.py:send():379] send: footer +2025-01-21 20:45:55,465 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:55,465 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 20:45:55,466 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:45:55,466 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:45:55,466 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 20:45:55,466 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:45:55,466 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: server_info +2025-01-21 20:45:55,467 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 20:45:55,468 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 20:45:55,469 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:55,530 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:45:55,530 INFO MainThread:2639291 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 20:45:55,530 INFO MainThread:2639291 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 20:45:55,531 INFO MainThread:2639291 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 20:45:55,531 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 20:45:55,531 INFO HandlerThread:2639291 [handler.py:finish():882] shutting down handler +2025-01-21 20:45:56,467 INFO WriterThread:2639291 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/run-07nra58y.wandb +2025-01-21 20:45:56,530 INFO SenderThread:2639291 [sender.py:finish():1615] shutting down sender +2025-01-21 20:45:56,530 INFO SenderThread:2639291 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 20:45:56,530 INFO SenderThread:2639291 [file_pusher.py:join():175] waiting for file pusher diff --git a/07nra58y/wandb/debug.log b/07nra58y/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..cc7abdd049d33997d358c8dd584e6168d967c245 --- /dev/null +++ b/07nra58y/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 20:42:00,335 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 20:42:00,335 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Configure stats pid to 2639119 +2025-01-21 20:42:00,335 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/settings +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/logs/debug.log +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/logs/debug-internal.log +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_init.py:init():569] calling init triggers +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y'} +2025-01-21 20:42:00,337 INFO MainThread:2639119 [wandb_init.py:init():619] starting backend +2025-01-21 20:42:00,337 INFO MainThread:2639119 [wandb_init.py:init():623] setting up manager +2025-01-21 20:42:00,344 INFO MainThread:2639119 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 20:42:00,345 INFO MainThread:2639119 [wandb_init.py:init():631] backend started and connected +2025-01-21 20:42:00,357 INFO MainThread:2639119 [wandb_init.py:init():720] updated telemetry +2025-01-21 20:42:00,363 INFO MainThread:2639119 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 20:42:00,688 INFO MainThread:2639119 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 20:42:00,758 INFO MainThread:2639119 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 20:42:00,758 INFO MainThread:2639119 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 20:42:01,102 INFO MainThread:2639119 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 20:42:01,103 INFO MainThread:2639119 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 20:42:01,104 INFO MainThread:2639119 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 20:42:01,104 INFO MainThread:2639119 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 20:42:01,108 INFO MainThread:2639119 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 20:42:01,108 INFO MainThread:2639119 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/tb, True +2025-01-21 20:42:05,088 INFO MainThread:2639119 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 20:45:56,531 WARNING MsgRouterThr:2639119 [router.py:message_loop():77] message_loop has been closed diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/files/code/FISH/eval_robot.py b/07nra58y/wandb/run-20250121_204200-07nra58y/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/files/config.yaml b/07nra58y/wandb/run-20250121_204200-07nra58y/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..793348426112f47fbba4f952b8da178d7075120a --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_0' +wandb_notes: + desc: null + value: 4295_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737510120 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/files/diff.patch b/07nra58y/wandb/run-20250121_204200-07nra58y/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..3a323adf2e00f53813f77e99fa964046e0cc21f4 --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/files/diff.patch @@ -0,0 +1,13 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..c2e87d3 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json b/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0ada6cde6d32ac525d48de8076e15c8904b601f9 --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[0, 5, 0.0, 0, 0.5534158944976787, 0, 0.515846252438237, 0, 0.3141024113073212, 0, 0.3513819575606989]]} \ No newline at end of file diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 b/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..88683a0a01cd11a49f9b34beaa830e09f7d03a41 --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4947144f835c80690116d2cb0e8332f4d2ceca6ec5bfb8c58a4b4f20ee84417 +size 633108 diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 b/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5bf51f3163afee71ce26507fa8ae2034832f63d3 --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d488251d8ba1dc3b3f72748124f7dbac02cf3572014e65f6ea0d79d795b00547 +size 675044 diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 b/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8e55762771773641ca1779b8034ef7c91ef5f4d9 --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c909f78c2729a4b3bd82117634333baf6c377e4bff3bedb8f26f8c7b7aae8a97 +size 677702 diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 b/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0296611c4966d387f745d6b350feaeb727b6c73b --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f946236f6700e74eb861ec8865fe687033f8099ed566b6d47b52734366fe6f7 +size 786909 diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 b/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b4dc940a829fc4523e8ccefc79fb383106ee7b26 --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd3afb63cd85bd424ac4f2182808e18600f37d681ebff8a80d3dd694c1c98823 +size 726949 diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log b/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..bbfddc72f8fe216ccfc29efbc9f36dfb63b76ef9 --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log @@ -0,0 +1,292 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737510125.088981]: resetting environment +[INFO] [1737510125.093118]: cleared current plan +[INFO] [1737510125.094044]: moving to home +[INFO] [1737510127.097069]: reached home +[INFO] [1737510127.098290]: reset action history +[INFO] [1737510129.814300]: environment reset +Starting episode 0 +[INFO] [1737510129.818452]: resetting environment +[INFO] [1737510129.822728]: cleared current plan +[INFO] [1737510129.823560]: moving to home +[INFO] [1737510130.825229]: reached home +[INFO] [1737510130.827029]: reset action history +[INFO] [1737510133.541583]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737510171.855604]: resetting environment +[INFO] [1737510171.860348]: cleared current plan +[INFO] [1737510171.861149]: moving to home +[INFO] [1737510172.863349]: reached home +[INFO] [1737510172.866743]: reset action history +[INFO] [1737510175.575109]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/episode_rosbags/episode_0_2025-01-21-20-42-55.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2639119.log +started roslaunch server http://158.130.50.37:43653/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2639411] +process[print_text-2]: started with pid [2639435] +process[rosbag_record-3]: started with pid [2639436] +started bagging! +For topic gripper_width: timestamp difference is 109842114 for nearest: 1737510188179201893 - target: 1737510188069359779 at idx 168 +preempting episode +[INFO] [1737510188.913670]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1737510189.666669]: Stored episode 1. +[INFO] [1737510189.666955]: Saving video... +[INFO] [1737510189.870266]: Video saved! +Starting episode 1 +[INFO] [1737510189.902653]: resetting environment +[INFO] [1737510189.912989]: cleared current plan +[INFO] [1737510189.913287]: moving to home +[INFO] [1737510193.216581]: reached home +[INFO] [1737510193.217216]: reset action history +[INFO] [1737510195.927772]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737510216.759924]: resetting environment +[INFO] [1737510216.763743]: cleared current plan +[INFO] [1737510216.763931]: moving to home +[INFO] [1737510217.766536]: reached home +[INFO] [1737510217.767126]: reset action history +[INFO] [1737510220.474828]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/episode_rosbags/episode_1_2025-01-21-20-43-40.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2639119.log +started roslaunch server http://158.130.50.37:33105/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2639569] +process[print_text-5]: started with pid [2639570] +process[rosbag_record-6]: started with pid [2639594] +started bagging! +For topic gripper_width: timestamp difference is 103273976 for nearest: 1737510221312540877 - target: 1737510221209266901 at idx 9 +preempting episode +[INFO] [1737510234.155937]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1737510234.872272]: Stored episode 2. +[INFO] [1737510234.872538]: Saving video... +[INFO] [1737510235.182951]: Video saved! +Starting episode 2 +[INFO] [1737510235.218210]: resetting environment +[INFO] [1737510235.235765]: cleared current plan +[INFO] [1737510235.236364]: moving to home +[INFO] [1737510238.741863]: reached home +[INFO] [1737510238.744070]: reset action history +[INFO] [1737510241.525098]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737510246.786096]: resetting environment +[INFO] [1737510246.795939]: cleared current plan +[INFO] [1737510246.796207]: moving to home +[INFO] [1737510247.797262]: reached home +[INFO] [1737510247.797691]: reset action history +[INFO] [1737510250.506245]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/episode_rosbags/episode_2_2025-01-21-20-44-10.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2639119.log +started roslaunch server http://158.130.50.37:46667/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2639743] +process[print_text-8]: started with pid [2639744] +process[rosbag_record-9]: started with pid [2639745] +started bagging! +For topic gripper_width: timestamp difference is 103988247 for nearest: 1737510260779183464 - target: 1737510260675195217 at idx 148 +preempting episode +[INFO] [1737510263.977297]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit[print_text-8] killing on exit +[INFO] [1737510264.807429]: Stored episode 3. +[INFO] [1737510264.813656]: Saving video... +[WARN] [1737510265.222197]: Plan exhausted +[WARN] [1737510265.245787]: Plan exhausted +[WARN] [1737510265.288242]: Plan exhausted +[WARN] [1737510265.326925]: Plan exhausted +[WARN] [1737510265.365811]: Plan exhausted +[INFO] [1737510265.292946]: Video saved! +Starting episode 3 +[INFO] [1737510265.333809]: resetting environment +[INFO] [1737510265.370018]: cleared current plan +[INFO] [1737510265.372281]: moving to home +[INFO] [1737510268.880928]: reached home +[INFO] [1737510268.884827]: reset action history +[INFO] [1737510271.661275]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737510276.119654]: resetting environment +[INFO] [1737510276.129001]: cleared current plan +[INFO] [1737510276.129585]: moving to home +[INFO] [1737510277.130298]: reached home +[INFO] [1737510277.131107]: reset action history +[INFO] [1737510279.857696]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/episode_rosbags/episode_3_2025-01-21-20-44-39.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2639119.log +started roslaunch server http://158.130.50.37:38285/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2639886] +process[print_text-11]: started with pid [2639887] +process[rosbag_record-12]: started with pid [2639888] +started bagging! +For topic gripper_width: timestamp difference is 118100209 for nearest: 1737510280879181905 - target: 1737510280761081696 at idx 15 +[WARN] [1737510296.686386]: Plan exhausted +[WARN] [1737510296.727114]: Plan exhausted +preempting episode +[INFO] [1737510295.489009]: Storing episode... +[print_text-11] killing on exit +[pub_text-10] killing on exit +[rosbag_record-12] killing on exit +[INFO] [1737510296.273422]: Stored episode 4. +[INFO] [1737510296.277518]: Saving video... +[INFO] [1737510296.953972]: Video saved! +[WARN] [1737510296.766716]: Plan exhausted +[WARN] [1737510296.806534]: Plan exhausted +[WARN] [1737510296.846054]: Plan exhausted +[WARN] [1737510296.886844]: Plan exhausted +[WARN] [1737510296.927765]: Plan exhausted +[WARN] [1737510296.966791]: Plan exhausted +[WARN] [1737510297.006325]: Plan exhausted +Starting episode 4 +[INFO] [1737510296.994940]: resetting environment +[INFO] [1737510297.028847]: cleared current plan +[INFO] [1737510297.031116]: moving to home +[INFO] [1737510300.546711]: reached home +[INFO] [1737510300.550971]: reset action history +[INFO] [1737510303.334114]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737510303.557731]: resetting environment +[INFO] [1737510303.564873]: cleared current plan +[INFO] [1737510303.565083]: moving to home +[INFO] [1737510304.566627]: reached home +[INFO] [1737510304.568568]: reset action history +[INFO] [1737510307.356905]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/episode_rosbags/episode_4_2025-01-21-20-45-07.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2639119.log +started roslaunch server http://158.130.50.37:38547/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2640029] +process[print_text-14]: started with pid [2640030] +process[rosbag_record-15]: started with pid [2640031] +started bagging! +preempting episode +[INFO] [1737510321.681264]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1737510322.575090]: Stored episode 5. +[INFO] [1737510322.577024]: Saving video... +[WARN] [1737510322.927608]: Plan exhausted +[WARN] [1737510322.965790]: Plan exhausted +[WARN] [1737510323.008759]: Plan exhausted +[WARN] [1737510323.045977]: Plan exhausted +[WARN] [1737510323.086627]: Plan exhausted +[WARN] [1737510323.133753]: Plan exhausted +[WARN] [1737510323.168416]: Plan exhausted +[WARN] [1737510323.206661]: Plan exhausted +[INFO] [1737510323.125468]: Video saved! +[INFO] [1737510323.198363]: resetting environment +[INFO] [1737510323.231398]: cleared current plan +[INFO] [1737510323.231776]: moving to home +[INFO] [1737510326.736452]: reached home +[INFO] [1737510326.739727]: reset action history +[INFO] [1737510329.525054]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/files/requirements.txt b/07nra58y/wandb/run-20250121_204200-07nra58y/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-metadata.json b/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..94f879a59aeb98ed2185a35b8b4b5f08028dead2 --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T01:42:00.781581", + "startedAt": "2025-01-22T01:42:00.329941", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3937.0602499999995, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4495.897, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.985, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 311.03546142578125 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json b/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6b50c81a903a9dee8bb0087bd4a743d6a66da95f --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "c4947144f835c80690116d2cb0e8332f4d2ceca6ec5bfb8c58a4b4f20ee84417", "size": 633108, "path": "media/videos/eval/0_eval_0_c4947144f835c8069011.mp4"}, "global_step": 259, "_timestamp": 1737510331.8538632, "_runtime": 211.50807213783264, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "d488251d8ba1dc3b3f72748124f7dbac02cf3572014e65f6ea0d79d795b00547", "size": 675044, "path": "media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4"}, "eval/num_success": 0.0, "episode": 5.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "c909f78c2729a4b3bd82117634333baf6c377e4bff3bedb8f26f8c7b7aae8a97", "size": 677702, "path": "media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "5f946236f6700e74eb861ec8865fe687033f8099ed566b6d47b52734366fe6f7", "size": 786909, "path": "media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "dd3afb63cd85bd424ac4f2182808e18600f37d681ebff8a80d3dd694c1c98823", "size": 726949, "path": "media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://2jjguvgw56zem76zax5q7nsgezkkfzyrm30vmd009yic2hg5yehp3ews8jwql5blha0gr50t241bswic7pg551m6sb4rlfoubsc4cy2l2sbcnbckq5hilsw1ltbwdh5k/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://gq1cbjle9hg6b5jpuau52dmbug65mbd35p1e9q7dfv06kybrbzqz96i1h664rsv8wxa9dpxwnbs37hza8tbh25dcczjtcgvqrmfya7fqom3wawoesklrrqyslnoyf4sw:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 0, "total_num_episodes": 5, "total_success_rate": 0.0, "total_success_rate_lb_0.01": 0, "total_success_rate_ub_0.01": 0.5534158944976787, "total_success_rate_lb_0.025": 0, "total_success_rate_ub_0.025": 0.515846252438237, "total_success_rate_lb_0.05": 0, "total_success_rate_ub_0.05": 0.3141024113073212, "total_success_rate_lb_0.1": 0, "total_success_rate_ub_0.1": 0.3513819575606989, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "3fbfe773b6a6adb9e9f84231abc953c71bfff9eaa3d69b2bf95911c7d2f3a912", "size": 434, "artifact_path": "wandb-client-artifact://4q601gfocyn95uzji645t9h1t29e9hju58b3mof116spd3i07jiti5h1rs8gjkds741hu752t0pgzah04vzrp6yhxzputdcfj8diev9bzbmi44hp961ktgusm441wwow/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://d5t8k0ns0dt18y9g42nw941n3pazo528f14v98tjmgu10z2qqdwp2kc4ehh9yniwv1he3ypk5f3m4msiwtaul8fqwhq2oozywwtvbgtb3d5iu6ohs9cgdsucrzzjru4a:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 224}, "eval/total_success_rate": 0.0, "eval/total_success_rate_lb_0.01": 0.0, "eval/total_success_rate_ub_0.01": 0.5534158945083618, "eval/total_success_rate_lb_0.025": 0.0, "eval/total_success_rate_ub_0.025": 0.5158462524414062, "eval/total_success_rate_lb_0.05": 0.0, "eval/total_success_rate_ub_0.05": 0.3141024112701416, "eval/total_success_rate_lb_0.1": 0.0, "eval/total_success_rate_ub_0.1": 0.35138195753097534} \ No newline at end of file diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/logs/debug-internal.log b/07nra58y/wandb/run-20250121_204200-07nra58y/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f78741371e57d026be6c451821690b7f9789047a --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/logs/debug-internal.log @@ -0,0 +1,711 @@ +2025-01-21 20:42:00,346 INFO StreamThr :2639291 [internal.py:wandb_internal():85] W&B internal server running at pid: 2639291, started at: 2025-01-21 20:42:00.345235 +2025-01-21 20:42:00,347 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status +2025-01-21 20:42:00,357 INFO WriterThread:2639291 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/run-07nra58y.wandb +2025-01-21 20:42:00,358 DEBUG SenderThread:2639291 [sender.py:send():379] send: header +2025-01-21 20:42:00,366 DEBUG SenderThread:2639291 [sender.py:send():379] send: run +2025-01-21 20:42:00,672 INFO SenderThread:2639291 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files +2025-01-21 20:42:00,672 INFO SenderThread:2639291 [sender.py:_start_run_threads():1188] run started: 07nra58y with start time 1737510120.345791 +2025-01-21 20:42:00,689 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 20:42:00,689 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: check_version +2025-01-21 20:42:00,763 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 20:42:00,768 DEBUG HandlerThread:2639291 [system_info.py:__init__():26] System info init +2025-01-21 20:42:00,768 DEBUG HandlerThread:2639291 [system_info.py:__init__():41] System info init done +2025-01-21 20:42:00,768 INFO HandlerThread:2639291 [system_monitor.py:start():194] Starting system monitor +2025-01-21 20:42:00,768 INFO SystemMonitor:2639291 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 20:42:00,768 INFO HandlerThread:2639291 [system_monitor.py:probe():214] Collecting system info +2025-01-21 20:42:00,769 INFO SystemMonitor:2639291 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 20:42:00,769 INFO SystemMonitor:2639291 [interfaces.py:start():188] Started disk monitoring +2025-01-21 20:42:00,770 INFO SystemMonitor:2639291 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 20:42:00,771 INFO SystemMonitor:2639291 [interfaces.py:start():188] Started memory monitoring +2025-01-21 20:42:00,771 INFO SystemMonitor:2639291 [interfaces.py:start():188] Started network monitoring +2025-01-21 20:42:00,781 DEBUG HandlerThread:2639291 [system_info.py:probe():152] Probing system +2025-01-21 20:42:00,783 DEBUG HandlerThread:2639291 [system_info.py:_probe_git():137] Probing git +2025-01-21 20:42:00,788 DEBUG HandlerThread:2639291 [system_info.py:_probe_git():145] Probing git done +2025-01-21 20:42:00,788 DEBUG HandlerThread:2639291 [system_info.py:probe():200] Probing system done +2025-01-21 20:42:00,788 DEBUG HandlerThread:2639291 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T01:42:00.781581', 'startedAt': '2025-01-22T01:42:00.329941', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.0602499999995, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4495.897, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.985, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 311.03546142578125}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 20:42:00,788 INFO HandlerThread:2639291 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 20:42:00,788 INFO HandlerThread:2639291 [system_monitor.py:probe():227] Publishing system info +2025-01-21 20:42:00,788 DEBUG HandlerThread:2639291 [system_info.py:_save_code():44] Saving code +2025-01-21 20:42:00,794 DEBUG HandlerThread:2639291 [system_info.py:_save_code():67] Saving code done +2025-01-21 20:42:00,794 DEBUG HandlerThread:2639291 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 20:42:00,856 DEBUG HandlerThread:2639291 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 20:42:00,857 INFO HandlerThread:2639291 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 20:42:00,860 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:42:00,860 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 20:42:00,860 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 20:42:00,860 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 20:42:01,102 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 20:42:01,102 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: python_packages +2025-01-21 20:42:01,102 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:42:01,102 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:42:01,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:01,134 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/6znpn0sz-wandb-metadata.json +2025-01-21 20:42:01,145 INFO wandb-upload_2:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/z0y8zo0r-diff.patch +2025-01-21 20:42:01,145 INFO HandlerThread:2639291 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/tb" + save: true +} +control { + relay_id: "e16b6b475cee4a50a1ed47d57c820f7d" +} +_info { + stream_id: "07nra58y" +} + +2025-01-21 20:42:01,146 DEBUG HandlerThread:2639291 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 20:42:01,186 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/s878f3g8-code/FISH/eval_robot.py +2025-01-21 20:42:01,268 DEBUG SenderThread:2639291 [sender.py:send():379] send: telemetry +2025-01-21 20:42:01,268 DEBUG SenderThread:2639291 [sender.py:send():379] send: tbrecord +2025-01-21 20:42:01,269 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:42:01,269 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 with policy live +2025-01-21 20:42:01,469 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/js8hmcxv-events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:42:01,673 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-metadata.json +2025-01-21 20:42:01,673 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:42:01,674 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/code/FISH/eval_robot.py +2025-01-21 20:42:01,674 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:01,674 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/requirements.txt +2025-01-21 20:42:01,674 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/diff.patch +2025-01-21 20:42:01,674 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/code +2025-01-21 20:42:01,674 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/code/FISH +2025-01-21 20:42:02,102 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:03,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:03,673 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:04,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:05,088 DEBUG SenderThread:2639291 [sender.py:send():379] send: config +2025-01-21 20:42:05,129 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:06,102 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:06,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:07,141 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:07,675 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:08,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:09,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:09,675 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:10,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:11,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:11,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:11,676 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:12,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:13,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:13,676 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:14,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:15,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:15,677 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:16,102 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:42:16,102 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:42:16,145 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:16,230 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:17,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:18,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:19,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:20,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:21,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:22,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:22,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:23,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:24,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:25,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:26,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:27,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:27,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:28,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:29,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:30,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:31,102 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:42:31,102 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:42:31,145 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:32,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:32,224 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:32,681 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/config.yaml +2025-01-21 20:42:33,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:34,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:35,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:36,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:37,107 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:38,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:38,107 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:39,107 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:40,107 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:41,107 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:42,107 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:43,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:43,107 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:44,108 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:45,108 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:46,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:42:46,103 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:42:46,145 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:47,108 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:48,108 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:48,185 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:49,108 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:50,108 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:51,108 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:52,109 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:53,109 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:53,687 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:53,905 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:54,109 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:55,109 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:55,687 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:56,110 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:57,110 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:57,688 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:42:58,110 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:42:59,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:42:59,110 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:00,112 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:00,772 DEBUG SystemMonitor:2639291 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 20:43:00,772 DEBUG SenderThread:2639291 [sender.py:send():379] send: stats +2025-01-21 20:43:01,102 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:43:01,103 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:43:01,145 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:02,112 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:03,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:04,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:04,170 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:05,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:06,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:07,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:08,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:09,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:09,667 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:09,691 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:09,900 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:43:09,900 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 with policy now +2025-01-21 20:43:09,900 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:43:09,901 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:43:09,901 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:43:09,902 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:43:10,114 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/pw6i8cez-media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 +2025-01-21 20:43:10,116 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:10,692 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:43:10,692 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 +2025-01-21 20:43:10,692 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos +2025-01-21 20:43:10,693 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval +2025-01-21 20:43:10,693 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media +2025-01-21 20:43:10,963 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/nf36eax3-media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 +2025-01-21 20:43:11,119 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:11,692 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:12,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:13,114 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:14,114 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:15,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:15,257 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:15,693 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:16,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:43:16,103 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:43:16,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:17,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:17,694 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:18,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:19,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:20,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:21,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:21,116 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:22,115 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:23,116 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:24,116 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:25,116 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:26,105 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:26,116 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:27,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:28,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:29,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:30,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:30,773 DEBUG SenderThread:2639291 [sender.py:send():379] send: stats +2025-01-21 20:43:31,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:43:31,103 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:43:31,145 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:31,246 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:32,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:33,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:34,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:35,117 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:36,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:36,751 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:37,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:37,701 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:37,701 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:43:37,971 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/als8vce5-events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:43:38,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:39,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:39,701 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:40,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:41,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:41,702 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:42,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:42,369 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:43,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:43,703 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:44,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:45,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:46,103 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:43:46,104 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:43:46,145 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:47,119 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:48,119 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:48,189 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:49,119 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:50,120 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:51,120 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:52,120 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:53,120 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:54,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:43:54,120 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:55,152 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:55,229 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:43:55,229 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:43:55,229 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 with policy now +2025-01-21 20:43:55,230 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:43:55,230 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:43:55,230 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:43:55,441 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/rga3v44b-media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 +2025-01-21 20:43:55,706 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:43:55,707 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:55,707 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 +2025-01-21 20:43:55,707 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval +2025-01-21 20:43:55,964 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/duif5rw2-media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 +2025-01-21 20:43:56,120 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:57,121 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:57,707 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:58,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:59,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:43:59,708 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:43:59,789 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:00,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:00,773 DEBUG SenderThread:2639291 [sender.py:send():379] send: stats +2025-01-21 20:44:01,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:44:01,113 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:44:01,157 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:02,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:03,124 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:03,709 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:04,124 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:05,124 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:05,570 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:06,124 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:07,124 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:07,571 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:44:07,571 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:44:07,571 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:44:07,711 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:44:07,711 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:07,712 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:44:08,124 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:09,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:09,712 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:10,124 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:10,838 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:11,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:11,713 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:12,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:13,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:13,714 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:14,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:15,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:16,104 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:44:16,104 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:44:16,145 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:16,259 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:17,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:18,125 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:19,128 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:20,128 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:21,129 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:21,716 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:22,111 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:22,129 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:23,129 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:24,129 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:25,131 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:25,326 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:44:25,326 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 with policy now +2025-01-21 20:44:25,329 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:44:25,329 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:44:25,329 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:44:25,330 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:44:25,564 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/bojhi70h-media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 +2025-01-21 20:44:25,718 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:44:25,718 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:25,718 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 +2025-01-21 20:44:25,718 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval +2025-01-21 20:44:26,016 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/8woarlbs-media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 +2025-01-21 20:44:26,132 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:27,114 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:27,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:27,718 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:28,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:29,132 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:29,719 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:30,142 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:30,774 DEBUG SenderThread:2639291 [sender.py:send():379] send: stats +2025-01-21 20:44:31,121 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:44:31,121 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:44:31,165 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:32,133 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:32,713 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:33,133 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:33,720 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:34,135 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:35,134 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:36,137 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:36,386 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:44:36,386 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:44:36,387 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:44:36,721 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:44:36,722 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:44:37,173 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:37,722 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:38,138 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:38,173 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:39,134 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:39,722 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:40,134 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:41,135 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:41,723 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:42,135 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:43,135 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:44,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:44,135 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:45,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:46,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:44:46,106 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:44:46,149 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:47,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:48,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:49,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:49,261 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:50,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:51,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:52,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:53,136 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:54,137 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:55,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:44:55,137 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:56,146 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:56,989 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:44:56,989 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 with policy now +2025-01-21 20:44:56,991 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:44:56,992 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:44:56,992 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:44:56,992 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:44:57,138 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:57,205 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/s7ru733e-media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 +2025-01-21 20:44:57,729 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:44:57,729 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:44:57,729 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 +2025-01-21 20:44:57,730 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval +2025-01-21 20:44:57,995 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/sgrw391v-media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 +2025-01-21 20:44:58,141 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:59,147 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:44:59,730 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:00,146 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:00,550 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:00,774 DEBUG SenderThread:2639291 [sender.py:send():379] send: stats +2025-01-21 20:45:01,113 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:45:01,114 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:45:01,154 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:01,730 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:02,150 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:03,157 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:03,731 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:45:03,984 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/osryolr7-events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:45:04,155 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:04,420 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:45:04,420 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:04,421 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:04,732 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:45:05,150 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:05,613 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:05,732 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:06,153 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:07,152 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:08,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:09,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:09,733 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:10,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:10,838 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:11,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:12,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:13,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:14,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:15,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:16,106 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:45:16,106 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:45:16,151 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:16,215 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:17,152 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:18,152 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:19,152 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:20,152 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:21,153 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:21,662 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:22,153 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:23,155 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:23,213 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:45:23,213 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:45:23,213 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 with policy now +2025-01-21 20:45:23,214 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:45:23,214 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:23,214 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:23,430 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/glti81mu-media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 +2025-01-21 20:45:23,738 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:45:23,738 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:23,738 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 +2025-01-21 20:45:23,738 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval +2025-01-21 20:45:23,999 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/dlrg1wek-media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 +2025-01-21 20:45:24,167 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:25,167 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:25,739 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:26,170 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:26,738 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:27,173 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:27,739 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:28,169 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:29,169 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:30,167 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:30,774 DEBUG SenderThread:2639291 [sender.py:send():379] send: stats +2025-01-21 20:45:31,114 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:45:31,114 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:45:31,164 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:31,426 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 20:45:31,426 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 20:45:31,606 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:45:31,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:31,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:45:31,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 20:45:31,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media +2025-01-21 20:45:31,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval +2025-01-21 20:45:31,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table +2025-01-21 20:45:31,925 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/gtcdtkfg-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 20:45:31,981 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 20:45:31,987 INFO wandb-upload_0:2639291 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp2he6lwp_ +2025-01-21 20:45:31,990 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:45:32,163 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:32,645 INFO SenderThread:2639291 [sender.py:send_request_log_artifact():1525] logged artifact run-07nra58y-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxODkzNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4NjEwMg==', 'latestArtifact': None}} +2025-01-21 20:45:32,645 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:45:32,645 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,646 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:32,646 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,646 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:45:32,646 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-21 20:45:32,647 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:45:32,647 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,647 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,647 DEBUG SenderThread:2639291 [sender.py:send():379] send: telemetry +2025-01-21 20:45:32,647 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,648 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,648 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,650 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,650 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,650 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,650 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,651 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,651 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,651 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,651 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,652 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,652 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,652 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,652 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,653 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,653 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,653 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,653 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,654 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,654 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:32,654 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:32,654 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 20:45:32,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:45:32,741 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:45:32,742 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-21 20:45:32,742 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval +2025-01-21 20:45:32,916 INFO wandb-upload_2:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/jtt99ao5-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 20:45:32,937 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/njtoi5mo-media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-21 20:45:33,022 INFO wandb-upload_1:2639291 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1fzi00cp +2025-01-21 20:45:33,163 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:33,429 INFO SenderThread:2639291 [sender.py:send_request_log_artifact():1525] logged artifact run-07nra58y-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxODk2NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4NjEwNg==', 'latestArtifact': None}} +2025-01-21 20:45:33,429 DEBUG SenderThread:2639291 [sender.py:send():379] send: files +2025-01-21 20:45:33,429 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json with policy now +2025-01-21 20:45:33,429 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:45:33,430 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:33,430 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:33,688 INFO wandb-upload_2:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/5yww2g8g-media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-21 20:45:33,742 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:45:33,742 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:34,163 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:35,163 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:36,163 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:37,163 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:38,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:38,163 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:39,164 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:40,164 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:41,164 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:42,164 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:43,118 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:43,164 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:44,165 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:44,837 DEBUG SenderThread:2639291 [sender.py:send():379] send: exit +2025-01-21 20:45:44,837 INFO SenderThread:2639291 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 20:45:44,838 INFO SenderThread:2639291 [sender.py:send_exit():588] handling runtime: 224 +2025-01-21 20:45:44,838 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:44,838 INFO SenderThread:2639291 [sender.py:send_exit():594] send defer +2025-01-21 20:45:44,838 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:44,838 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 20:45:44,838 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:44,838 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 20:45:44,838 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 1 +2025-01-21 20:45:44,839 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:44,839 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 20:45:44,839 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:44,839 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 20:45:44,839 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 2 +2025-01-21 20:45:44,839 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:44,839 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 20:45:44,839 INFO HandlerThread:2639291 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 20:45:44,839 DEBUG SystemMonitor:2639291 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 20:45:44,840 DEBUG SystemMonitor:2639291 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 20:45:44,840 INFO HandlerThread:2639291 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 20:45:44,840 INFO HandlerThread:2639291 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 20:45:44,892 INFO HandlerThread:2639291 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 20:45:44,893 INFO HandlerThread:2639291 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 20:45:44,893 INFO HandlerThread:2639291 [interfaces.py:finish():200] Joined network monitor +2025-01-21 20:45:44,893 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:44,893 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 20:45:44,893 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 3 +2025-01-21 20:45:44,893 DEBUG SenderThread:2639291 [sender.py:send():379] send: stats +2025-01-21 20:45:44,894 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:44,894 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 20:45:44,894 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:44,894 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 20:45:44,894 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 4 +2025-01-21 20:45:44,894 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:44,894 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 20:45:45,746 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:45:45,747 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:49,748 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/config.yaml +2025-01-21 20:45:51,462 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:45:51,462 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:51,462 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 20:45:51,463 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:51,463 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 20:45:51,463 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 5 +2025-01-21 20:45:51,463 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:45:51,464 DEBUG SenderThread:2639291 [sender.py:send():379] send: history +2025-01-21 20:45:51,464 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:51,464 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:45:51,464 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 20:45:51,465 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:51,465 DEBUG SenderThread:2639291 [sender.py:send():379] send: summary +2025-01-21 20:45:51,469 INFO SenderThread:2639291 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:45:51,469 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:51,469 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 20:45:51,469 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 6 +2025-01-21 20:45:51,469 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:51,469 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 20:45:51,469 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:51,469 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 20:45:51,469 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 7 +2025-01-21 20:45:51,470 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:45:51,470 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:51,470 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 20:45:51,470 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:51,470 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 20:45:51,753 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:45:51,839 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:45:53,305 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 8 +2025-01-21 20:45:53,305 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:45:53,305 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:53,305 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 20:45:53,306 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:53,306 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 20:45:53,306 INFO SenderThread:2639291 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 20:45:53,308 INFO SenderThread:2639291 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 20:45:53,310 INFO SenderThread:2639291 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 20:45:53,324 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 9 +2025-01-21 20:45:53,324 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:53,324 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 20:45:53,325 DEBUG SenderThread:2639291 [sender.py:send():379] send: artifact +2025-01-21 20:45:53,750 INFO Thread-12 :2639291 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:53,771 INFO wandb-upload_0:2639291 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6xpc5ztm +2025-01-21 20:45:53,839 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:45:53,899 INFO wandb-upload_1:2639291 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1__rqxph +2025-01-21 20:45:53,914 INFO wandb-upload_2:2639291 [upload_job.py:push():88] Uploaded file /tmp/tmp1vwz0ran/wandb-job.json +2025-01-21 20:45:54,450 INFO SenderThread:2639291 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYxNDI3MQ==', 'versionIndex': 452}}} +2025-01-21 20:45:54,450 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:54,450 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 20:45:54,450 INFO SenderThread:2639291 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 20:45:54,751 INFO SenderThread:2639291 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files +2025-01-21 20:45:54,751 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/config.yaml config.yaml +2025-01-21 20:45:54,751 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json wandb-summary.json +2025-01-21 20:45:54,751 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/requirements.txt requirements.txt +2025-01-21 20:45:54,751 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-metadata.json wandb-metadata.json +2025-01-21 20:45:54,751 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:45:54,753 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log output.log +2025-01-21 20:45:54,754 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/diff.patch diff.patch +2025-01-21 20:45:54,756 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 20:45:54,756 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 media/videos/eval/0_eval_0_c4947144f835c8069011.mp4 +2025-01-21 20:45:54,756 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 media/videos/eval/3_eval_3_5f946236f6700e74eb86.mp4 +2025-01-21 20:45:54,757 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 media/videos/eval/1_eval_1_d488251d8ba1dc3b3f72.mp4 +2025-01-21 20:45:54,757 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 media/videos/eval/2_eval_2_c909f78c2729a4b3bd82.mp4 +2025-01-21 20:45:54,757 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 media/videos/eval/4_eval_4_dd3afb63cd85bd424ac4.mp4 +2025-01-21 20:45:54,757 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-21 20:45:54,757 INFO SenderThread:2639291 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-21 20:45:54,757 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 10 +2025-01-21 20:45:54,757 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:45:54,759 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:54,760 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 20:45:54,761 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:54,761 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 20:45:54,761 INFO SenderThread:2639291 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 20:45:54,840 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:45:54,840 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:45:54,939 INFO wandb-upload_0:2639291 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/config.yaml +2025-01-21 20:45:54,949 INFO wandb-upload_2:2639291 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/requirements.txt +2025-01-21 20:45:54,951 INFO wandb-upload_1:2639291 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/wandb-summary.json +2025-01-21 20:45:55,003 INFO wandb-upload_4:2639291 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/files/output.log +2025-01-21 20:45:55,059 INFO wandb-upload_3:2639291 [upload_job.py:push():130] Uploaded file /tmp/tmp34vccjw9wandb/lqk4vyyj-events.out.tfevents.1737510121.leonmkim-ROG-Strix-G15CS-G15CS.2639119.0 +2025-01-21 20:45:55,259 INFO Thread-11 (_thread_body):2639291 [sender.py:transition_state():614] send defer: 11 +2025-01-21 20:45:55,259 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:55,259 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 20:45:55,260 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:55,260 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 20:45:55,260 INFO SenderThread:2639291 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 20:45:55,261 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 12 +2025-01-21 20:45:55,261 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:55,261 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 20:45:55,262 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:55,262 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 20:45:55,262 INFO SenderThread:2639291 [file_stream.py:finish():601] file stream finish called +2025-01-21 20:45:55,464 INFO SenderThread:2639291 [file_stream.py:finish():605] file stream finish is done +2025-01-21 20:45:55,464 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 13 +2025-01-21 20:45:55,464 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:55,464 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 20:45:55,464 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:55,465 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 20:45:55,465 INFO SenderThread:2639291 [sender.py:transition_state():614] send defer: 14 +2025-01-21 20:45:55,465 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:45:55,465 INFO HandlerThread:2639291 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 20:45:55,465 DEBUG SenderThread:2639291 [sender.py:send():379] send: final +2025-01-21 20:45:55,465 DEBUG SenderThread:2639291 [sender.py:send():379] send: footer +2025-01-21 20:45:55,465 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: defer +2025-01-21 20:45:55,465 INFO SenderThread:2639291 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 20:45:55,466 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:45:55,466 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:45:55,466 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 20:45:55,466 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:45:55,466 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: server_info +2025-01-21 20:45:55,467 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 20:45:55,468 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 20:45:55,469 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:45:55,530 DEBUG SenderThread:2639291 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:45:55,530 INFO MainThread:2639291 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 20:45:55,530 INFO MainThread:2639291 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 20:45:55,531 INFO MainThread:2639291 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 20:45:55,531 DEBUG HandlerThread:2639291 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 20:45:55,531 INFO HandlerThread:2639291 [handler.py:finish():882] shutting down handler +2025-01-21 20:45:56,467 INFO WriterThread:2639291 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/run-07nra58y.wandb +2025-01-21 20:45:56,530 INFO SenderThread:2639291 [sender.py:finish():1615] shutting down sender +2025-01-21 20:45:56,530 INFO SenderThread:2639291 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 20:45:56,530 INFO SenderThread:2639291 [file_pusher.py:join():175] waiting for file pusher diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/logs/debug.log b/07nra58y/wandb/run-20250121_204200-07nra58y/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..cc7abdd049d33997d358c8dd584e6168d967c245 --- /dev/null +++ b/07nra58y/wandb/run-20250121_204200-07nra58y/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 20:42:00,335 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 20:42:00,335 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Configure stats pid to 2639119 +2025-01-21 20:42:00,335 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/settings +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/logs/debug.log +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/wandb/run-20250121_204200-07nra58y/logs/debug-internal.log +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_init.py:init():569] calling init triggers +2025-01-21 20:42:00,336 INFO MainThread:2639119 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y'} +2025-01-21 20:42:00,337 INFO MainThread:2639119 [wandb_init.py:init():619] starting backend +2025-01-21 20:42:00,337 INFO MainThread:2639119 [wandb_init.py:init():623] setting up manager +2025-01-21 20:42:00,344 INFO MainThread:2639119 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 20:42:00,345 INFO MainThread:2639119 [wandb_init.py:init():631] backend started and connected +2025-01-21 20:42:00,357 INFO MainThread:2639119 [wandb_init.py:init():720] updated telemetry +2025-01-21 20:42:00,363 INFO MainThread:2639119 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 20:42:00,688 INFO MainThread:2639119 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 20:42:00,758 INFO MainThread:2639119 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 20:42:00,758 INFO MainThread:2639119 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 20:42:01,102 INFO MainThread:2639119 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 20:42:01,103 INFO MainThread:2639119 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 20:42:01,104 INFO MainThread:2639119 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 20:42:01,104 INFO MainThread:2639119 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 20:42:01,108 INFO MainThread:2639119 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 20:42:01,108 INFO MainThread:2639119 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/07nra58y/tb, True +2025-01-21 20:42:05,088 INFO MainThread:2639119 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 20:45:56,531 WARNING MsgRouterThr:2639119 [router.py:message_loop():77] message_loop has been closed diff --git a/07nra58y/wandb/run-20250121_204200-07nra58y/run-07nra58y.wandb b/07nra58y/wandb/run-20250121_204200-07nra58y/run-07nra58y.wandb new file mode 100644 index 0000000000000000000000000000000000000000..18c8284935ae11d42276820015fcd98405cd82ae Binary files /dev/null and b/07nra58y/wandb/run-20250121_204200-07nra58y/run-07nra58y.wandb differ diff --git a/7uoahjwu/.hydra/config.yaml b/7uoahjwu/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90279a1d8d4077f1affa573bd8d5db0ff727e458 --- /dev/null +++ b/7uoahjwu/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/7uoahjwu/.hydra/hydra.yaml b/7uoahjwu/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..410f1aeeabec030a4e13c532c759651c4f1a718d --- /dev/null +++ b/7uoahjwu/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/7uoahjwu + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/7uoahjwu/.hydra/overrides.yaml b/7uoahjwu/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/7uoahjwu/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/7uoahjwu/eval_policy.log b/7uoahjwu/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..424478432222d8d6878b23fa65e2db9deda9789d --- /dev/null +++ b/7uoahjwu/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-21 20:39:59,431][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 20:39:59,435][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-21 20:40:00,535][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/aqyye28j/.hydra/config.yaml b/aqyye28j/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dc75d61dbe4b7ba0ede1ab77029649cc5b18502 --- /dev/null +++ b/aqyye28j/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/aqyye28j/.hydra/hydra.yaml b/aqyye28j/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f94f0dbdbf22ad847716e25c1aa1a528691f3a5 --- /dev/null +++ b/aqyye28j/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/aqyye28j/.hydra/overrides.yaml b/aqyye28j/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/aqyye28j/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/aqyye28j/episode_rosbags/episode_1_2025-01-21-22-08-24.bag b/aqyye28j/episode_rosbags/episode_1_2025-01-21-22-08-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..88fb717e6a22c9c0101eaf1c0c4e66e468c255f7 --- /dev/null +++ b/aqyye28j/episode_rosbags/episode_1_2025-01-21-22-08-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18b3d55caeb24b0d0656de2a9092222ee8c55bd93dae22570f1da2b6651337de +size 1128555325 diff --git a/aqyye28j/episode_rosbags/episode_2_2025-01-21-22-08-51.bag b/aqyye28j/episode_rosbags/episode_2_2025-01-21-22-08-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..da4f300b8756f8fbb4567eaa63662b3ee4579adf --- /dev/null +++ b/aqyye28j/episode_rosbags/episode_2_2025-01-21-22-08-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a64d3130a2750a1fe2c5c47210e4cd6eda3691b708332e828e411ef374ff714b +size 1073491052 diff --git a/aqyye28j/eval_video/0_eval.mp4 b/aqyye28j/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..07ab74cf9dbce72408e70009d4cc9b7a06b7c67f --- /dev/null +++ b/aqyye28j/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51125b9dabf4e9b3cedcd36bf6f097b605281070efe383308692d343bdc07522 +size 1526251 diff --git a/aqyye28j/eval_video/1_eval.mp4 b/aqyye28j/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bb65a48e3a8f1d48d74184448288e387f725d204 --- /dev/null +++ b/aqyye28j/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27250e24fb2dabe74cd6138f841e48b5062f21ab0105d92770e2a1dc76782e8f +size 777927 diff --git a/aqyye28j/eval_video/2_eval.mp4 b/aqyye28j/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8b00ff0dbe184831ac7b4301aaa3b7c4e3821cfb --- /dev/null +++ b/aqyye28j/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96f346beb19c014ed98ce09ddc37c0e6bc4a2ec48377abfbcdedd9e65e8ed485 +size 744132 diff --git a/aqyye28j/eval_video/3_eval.mp4 b/aqyye28j/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f7d00b2bb2d3fab2aa7a2927051d5289c63c1b72 --- /dev/null +++ b/aqyye28j/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cfd6f1db2c9dcc9eb14fdd05e29177fb84a17d1241df2bccda3ec99c472c710 +size 1566794 diff --git a/aqyye28j/eval_video/4_eval.mp4 b/aqyye28j/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a967b97da8df2f85f599f9c025ae76f213020d1c --- /dev/null +++ b/aqyye28j/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6afd64f2a3f512f6d9a67af92605d237cd01284da9aacd680b56871000a9afbe +size 1532772 diff --git a/aqyye28j/tb/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 b/aqyye28j/tb/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 new file mode 100644 index 0000000000000000000000000000000000000000..58f86e361b13975fbc59e5ef4de48fc6cf7706c2 --- /dev/null +++ b/aqyye28j/tb/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e271ccfeda8e82808cb72410f86ffa33acee550d7e5b4597f60974ea1a1acd9d +size 6539 diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a967b97da8df2f85f599f9c025ae76f213020d1c --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6afd64f2a3f512f6d9a67af92605d237cd01284da9aacd680b56871000a9afbe +size 1532772 diff --git a/aqyye28j/wandb/run-20250121_220708-aqyye28j/logs/debug-internal.log b/aqyye28j/wandb/run-20250121_220708-aqyye28j/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3ba7297175f55ecba0d939a4bd57721161c2de77 --- /dev/null +++ b/aqyye28j/wandb/run-20250121_220708-aqyye28j/logs/debug-internal.log @@ -0,0 +1,759 @@ +2025-01-21 22:07:08,229 INFO StreamThr :2661581 [internal.py:wandb_internal():85] W&B internal server running at pid: 2661581, started at: 2025-01-21 22:07:08.229643 +2025-01-21 22:07:08,231 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status +2025-01-21 22:07:08,242 INFO WriterThread:2661581 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/run-aqyye28j.wandb +2025-01-21 22:07:08,243 DEBUG SenderThread:2661581 [sender.py:send():379] send: header +2025-01-21 22:07:08,252 DEBUG SenderThread:2661581 [sender.py:send():379] send: run +2025-01-21 22:07:08,592 INFO SenderThread:2661581 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files +2025-01-21 22:07:08,592 INFO SenderThread:2661581 [sender.py:_start_run_threads():1188] run started: aqyye28j with start time 1737515228.229771 +2025-01-21 22:07:08,607 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 22:07:08,607 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: check_version +2025-01-21 22:07:08,686 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 22:07:08,691 DEBUG HandlerThread:2661581 [system_info.py:__init__():26] System info init +2025-01-21 22:07:08,691 DEBUG HandlerThread:2661581 [system_info.py:__init__():41] System info init done +2025-01-21 22:07:08,691 INFO HandlerThread:2661581 [system_monitor.py:start():194] Starting system monitor +2025-01-21 22:07:08,691 INFO SystemMonitor:2661581 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 22:07:08,691 INFO HandlerThread:2661581 [system_monitor.py:probe():214] Collecting system info +2025-01-21 22:07:08,692 INFO SystemMonitor:2661581 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 22:07:08,692 INFO SystemMonitor:2661581 [interfaces.py:start():188] Started disk monitoring +2025-01-21 22:07:08,693 INFO SystemMonitor:2661581 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 22:07:08,693 INFO SystemMonitor:2661581 [interfaces.py:start():188] Started memory monitoring +2025-01-21 22:07:08,693 INFO SystemMonitor:2661581 [interfaces.py:start():188] Started network monitoring +2025-01-21 22:07:08,705 DEBUG HandlerThread:2661581 [system_info.py:probe():152] Probing system +2025-01-21 22:07:08,706 DEBUG HandlerThread:2661581 [system_info.py:_probe_git():137] Probing git +2025-01-21 22:07:08,711 DEBUG HandlerThread:2661581 [system_info.py:_probe_git():145] Probing git done +2025-01-21 22:07:08,711 DEBUG HandlerThread:2661581 [system_info.py:probe():200] Probing system done +2025-01-21 22:07:08,711 DEBUG HandlerThread:2661581 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T03:07:08.705289', 'startedAt': '2025-01-22T03:07:08.219659', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4123.8076249999995, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.981, 'min': 800.0, 'max': 4700.0}, {'current': 4499.971, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 449.60555267333984}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 22:07:08,711 INFO HandlerThread:2661581 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 22:07:08,711 INFO HandlerThread:2661581 [system_monitor.py:probe():227] Publishing system info +2025-01-21 22:07:08,711 DEBUG HandlerThread:2661581 [system_info.py:_save_code():44] Saving code +2025-01-21 22:07:08,713 DEBUG HandlerThread:2661581 [system_info.py:_save_code():67] Saving code done +2025-01-21 22:07:08,713 DEBUG HandlerThread:2661581 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 22:07:08,776 DEBUG HandlerThread:2661581 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 22:07:08,776 INFO HandlerThread:2661581 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 22:07:08,780 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:07:08,780 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 22:07:08,780 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 22:07:08,780 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 22:07:09,018 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 22:07:09,018 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: python_packages +2025-01-21 22:07:09,018 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:07:09,019 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:07:09,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:09,022 INFO HandlerThread:2661581 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/tb" + save: true +} +control { + relay_id: "75b487f69feb46dca4f1836e763f05bd" +} +_info { + stream_id: "aqyye28j" +} + +2025-01-21 22:07:09,022 DEBUG HandlerThread:2661581 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 22:07:09,047 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/7pk3fx6n-wandb-metadata.json +2025-01-21 22:07:09,076 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/77h50yn7-code/FISH/eval_robot.py +2025-01-21 22:07:09,079 DEBUG SenderThread:2661581 [sender.py:send():379] send: telemetry +2025-01-21 22:07:09,079 DEBUG SenderThread:2661581 [sender.py:send():379] send: tbrecord +2025-01-21 22:07:09,080 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:07:09,080 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 with policy live +2025-01-21 22:07:09,088 INFO wandb-upload_2:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/6z2xrcrr-diff.patch +2025-01-21 22:07:09,277 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/gr073swi-events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:07:09,594 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/diff.patch +2025-01-21 22:07:09,594 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/code/FISH/eval_robot.py +2025-01-21 22:07:09,594 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/requirements.txt +2025-01-21 22:07:09,594 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-metadata.json +2025-01-21 22:07:09,594 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:07:09,594 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:09,594 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/code +2025-01-21 22:07:09,595 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/code/FISH +2025-01-21 22:07:10,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:11,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:11,593 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:12,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:13,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:14,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:14,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:15,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:16,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:16,715 DEBUG SenderThread:2661581 [sender.py:send():379] send: config +2025-01-21 22:07:17,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:18,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:19,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:19,595 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:20,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:20,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:21,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:21,596 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:22,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:23,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:23,597 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:24,018 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:07:24,019 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:07:24,061 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:25,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:25,209 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:25,597 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:26,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:27,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:28,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:29,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:30,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:31,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:31,065 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:31,599 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:32,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:33,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:33,600 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:34,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:35,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:35,600 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:36,023 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:36,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:37,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:38,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:39,018 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:07:39,018 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:07:39,061 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:39,601 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:07:40,023 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:41,023 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:41,162 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:41,602 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/config.yaml +2025-01-21 22:07:42,023 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:43,024 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:44,024 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:45,024 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:46,024 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:47,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:47,024 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:48,024 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:49,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:50,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:51,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:52,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:53,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:53,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:54,018 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:07:54,019 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:07:54,061 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:55,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:56,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:57,025 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:58,026 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:07:58,181 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:07:59,026 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:00,026 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:01,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:02,028 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:03,028 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:04,020 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:04,028 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:05,028 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:06,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:07,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:08,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:08,694 DEBUG SystemMonitor:2661581 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 22:08:08,694 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:08:09,018 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:08:09,019 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:08:09,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:09,165 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:09,826 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:08:09,827 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 with policy now +2025-01-21 22:08:09,827 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:08:09,828 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:08:09,828 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:08:09,829 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:08:10,030 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:10,109 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/8b7e7wby-media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 +2025-01-21 22:08:10,609 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 +2025-01-21 22:08:10,609 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:08:10,610 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos +2025-01-21 22:08:10,610 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media +2025-01-21 22:08:10,610 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval +2025-01-21 22:08:10,924 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/2it6cc87-media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 +2025-01-21 22:08:11,030 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:11,610 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:12,030 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:13,030 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:14,030 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:15,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:15,030 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:15,611 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:16,031 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:17,031 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:17,612 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:18,031 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:19,031 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:20,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:20,032 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:21,032 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:21,613 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:08:21,877 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/xjglzqfs-events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:08:22,032 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:23,032 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:23,614 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:24,018 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:08:24,019 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:08:24,061 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:25,028 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:25,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:26,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:27,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:27,615 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:28,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:29,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:30,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:31,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:31,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:32,039 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:33,039 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:34,039 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:35,039 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:36,022 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:36,039 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:37,040 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:38,040 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:38,695 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:08:39,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:08:39,019 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:08:39,061 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:40,040 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:41,041 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:41,284 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:41,443 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:08:41,443 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 with policy now +2025-01-21 22:08:41,446 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:08:41,446 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:08:41,446 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:08:41,447 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:08:41,620 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:08:41,620 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:41,620 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 +2025-01-21 22:08:41,620 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval +2025-01-21 22:08:41,703 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/ifuepsg8-media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 +2025-01-21 22:08:41,975 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/8xh0el6j-media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 +2025-01-21 22:08:42,043 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:43,043 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:43,620 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:44,043 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:45,046 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:46,042 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:47,005 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:47,042 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:47,622 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:48,043 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:48,412 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:08:48,412 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:08:48,413 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:08:48,622 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:08:48,622 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:08:49,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:49,622 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:50,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:51,043 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:51,623 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:52,007 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:52,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:53,043 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:53,624 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:08:54,019 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:08:54,019 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:08:54,065 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:55,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:56,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:57,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:57,162 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:08:58,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:08:59,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:00,044 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:01,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:01,626 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:02,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:03,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:03,125 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:04,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:05,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:06,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:07,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:07,529 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:09:07,529 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 with policy now +2025-01-21 22:09:07,530 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:09:07,530 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:09:07,531 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:09:07,533 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:09:07,628 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:09:07,629 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:07,629 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 +2025-01-21 22:09:07,629 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval +2025-01-21 22:09:07,800 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/em1fn7ja-media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 +2025-01-21 22:09:08,045 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:08,064 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/xkxneh5e-media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 +2025-01-21 22:09:08,560 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:08,695 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:09:09,036 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:09:09,037 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:09:09,081 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:09,629 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:10,049 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:11,059 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:12,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:13,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:13,630 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:13,806 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:14,048 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:15,049 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:15,631 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:09:15,631 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:16,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:16,224 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:09:16,224 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:09:16,225 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:09:16,631 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:09:17,049 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:17,632 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:18,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:19,032 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:19,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:20,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:21,051 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:21,633 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:22,051 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:23,051 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:24,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:09:24,021 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:09:24,061 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:24,086 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:25,051 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:26,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:27,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:28,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:29,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:30,026 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:30,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:31,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:32,052 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:33,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:34,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:35,026 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:35,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:36,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:37,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:38,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:38,696 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:09:39,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:09:39,021 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:09:39,061 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:40,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:40,101 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:41,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:42,053 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:43,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:44,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:45,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:46,027 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:46,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:47,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:48,054 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:49,055 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:49,642 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:50,056 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:51,028 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:51,056 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:52,056 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:53,056 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:54,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:09:54,021 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:09:54,065 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:55,057 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:56,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:56,411 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:09:56,618 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:09:56,618 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 with policy now +2025-01-21 22:09:56,624 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:09:56,624 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:09:56,624 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:09:56,632 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:09:56,645 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:09:56,645 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 +2025-01-21 22:09:56,645 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval +2025-01-21 22:09:56,917 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/ptqyh5vo-media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 +2025-01-21 22:09:57,058 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:57,214 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/d819ww7a-media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 +2025-01-21 22:09:57,646 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:09:58,062 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:09:59,075 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:00,064 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:01,063 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:01,648 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:02,065 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:02,317 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:03,065 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:04,064 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:04,649 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:10:04,910 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/fdx00fwz-events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:10:05,067 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:05,278 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:10:05,278 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:05,278 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:05,649 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:10:05,649 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:06,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:07,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:07,650 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:07,697 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:08,069 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:08,696 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:10:09,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:10:09,021 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:10:09,066 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:09,651 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:10,067 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:11,067 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:11,651 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:12,067 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:13,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:13,115 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:14,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:15,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:16,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:17,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:18,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:19,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:19,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:20,068 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:21,069 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:22,069 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:23,069 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:24,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:10:24,021 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:10:24,069 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:24,145 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:25,069 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:25,656 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:26,069 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:27,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:27,657 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:28,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:29,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:29,658 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:30,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:30,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:31,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:32,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:33,070 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:34,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:35,029 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:35,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:36,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:37,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:37,660 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:38,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:38,697 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:10:39,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:10:39,021 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:10:39,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:40,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:40,143 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:41,071 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:42,072 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:43,072 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:44,075 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:44,859 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:10:44,859 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 with policy now +2025-01-21 22:10:44,861 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:10:44,862 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:10:44,862 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:44,866 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:45,073 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:45,129 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/wwebsqsd-media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 +2025-01-21 22:10:45,663 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:10:45,663 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:45,664 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 +2025-01-21 22:10:45,664 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval +2025-01-21 22:10:45,888 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:45,976 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/koso4oum-media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 +2025-01-21 22:10:46,081 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:47,084 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:47,664 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:48,078 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:49,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:49,665 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:50,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:51,038 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:51,099 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:52,086 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:53,088 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:53,666 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:10:54,034 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:10:54,034 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:10:54,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:55,097 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:56,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:56,192 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:10:57,089 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:57,343 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:10:57,343 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:57,345 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:57,589 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:10:57,589 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:10:57,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:10:57,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:10:57,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-21 22:10:57,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media +2025-01-21 22:10:57,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table +2025-01-21 22:10:57,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval +2025-01-21 22:10:57,789 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:10:57,863 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/1x4ld4vr-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-21 22:10:58,071 INFO wandb-upload_0:2661581 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0d7hi5di +2025-01-21 22:10:58,082 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:58,236 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:10:58,247 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:10:58,522 INFO SenderThread:2661581 [sender.py:send_request_log_artifact():1525] logged artifact run-aqyye28j-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcyMDg0NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwNDgzMg==', 'latestArtifact': None}} +2025-01-21 22:10:58,522 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:10:58,522 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json with policy now +2025-01-21 22:10:58,522 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:10:58,522 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,523 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,523 DEBUG SenderThread:2661581 [sender.py:send():379] send: telemetry +2025-01-21 22:10:58,523 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,527 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,527 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,528 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,528 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,528 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,528 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,529 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,529 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,529 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,529 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,530 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,530 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,530 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,530 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,531 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,531 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,531 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,531 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,532 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,532 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:58,532 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:58,532 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:10:58,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:10:58,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:10:58,669 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json +2025-01-21 22:10:58,670 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval +2025-01-21 22:10:58,786 INFO wandb-upload_2:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/6e4upphm-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-21 22:10:58,861 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/64lup3tl-media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json +2025-01-21 22:10:58,946 INFO wandb-upload_1:2661581 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpztmwj91e +2025-01-21 22:10:59,082 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:10:59,350 INFO SenderThread:2661581 [sender.py:send_request_log_artifact():1525] logged artifact run-aqyye28j-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcyMDg2Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwNDgzOA==', 'latestArtifact': None}} +2025-01-21 22:10:59,350 DEBUG SenderThread:2661581 [sender.py:send():379] send: files +2025-01-21 22:10:59,350 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json with policy now +2025-01-21 22:10:59,350 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:10:59,350 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:10:59,351 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:10:59,616 INFO wandb-upload_2:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/t5uoni3c-media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json +2025-01-21 22:10:59,670 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:10:59,670 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:11:00,082 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:01,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:01,351 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:02,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:03,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:04,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:05,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:06,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:07,033 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:07,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:08,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:08,697 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:11:09,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:11:09,021 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:11:09,083 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:10,084 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:11,084 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:12,084 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:12,148 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:13,084 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:14,084 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:15,084 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:16,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:17,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:18,034 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:18,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:19,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:20,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:21,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:22,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:23,034 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:23,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:24,021 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:11:24,022 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:11:24,085 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:25,086 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:26,086 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:27,086 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:28,086 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:28,182 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:28,421 DEBUG SenderThread:2661581 [sender.py:send():379] send: exit +2025-01-21 22:11:28,421 INFO SenderThread:2661581 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 22:11:28,421 INFO SenderThread:2661581 [sender.py:send_exit():588] handling runtime: 259 +2025-01-21 22:11:28,422 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:11:28,423 INFO SenderThread:2661581 [sender.py:send_exit():594] send defer +2025-01-21 22:11:28,423 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:28,423 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 22:11:28,423 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:28,423 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 22:11:28,423 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 1 +2025-01-21 22:11:28,423 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:28,423 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 22:11:28,423 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:28,423 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 22:11:28,423 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 2 +2025-01-21 22:11:28,424 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:28,424 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 22:11:28,424 INFO HandlerThread:2661581 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 22:11:28,424 INFO HandlerThread:2661581 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 22:11:28,424 DEBUG SystemMonitor:2661581 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 22:11:28,424 INFO HandlerThread:2661581 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 22:11:28,424 DEBUG SystemMonitor:2661581 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 22:11:28,477 INFO HandlerThread:2661581 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 22:11:28,477 INFO HandlerThread:2661581 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 22:11:28,477 INFO HandlerThread:2661581 [interfaces.py:finish():200] Joined network monitor +2025-01-21 22:11:28,477 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:28,477 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 22:11:28,477 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 3 +2025-01-21 22:11:28,478 DEBUG SenderThread:2661581 [sender.py:send():379] send: stats +2025-01-21 22:11:28,478 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:28,478 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 22:11:28,478 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:28,478 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 22:11:28,478 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 4 +2025-01-21 22:11:28,478 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:28,478 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 22:11:28,681 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/config.yaml +2025-01-21 22:11:28,682 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:11:29,682 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:11:35,352 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:11:35,353 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:35,353 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 22:11:35,354 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:35,354 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 22:11:35,354 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 5 +2025-01-21 22:11:35,354 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:35,354 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 22:11:35,354 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:11:35,355 DEBUG SenderThread:2661581 [sender.py:send():379] send: history +2025-01-21 22:11:35,355 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:11:35,355 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:11:35,355 DEBUG SenderThread:2661581 [sender.py:send():379] send: summary +2025-01-21 22:11:35,356 INFO SenderThread:2661581 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:11:35,356 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:35,356 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 22:11:35,356 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 6 +2025-01-21 22:11:35,356 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:35,356 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 22:11:35,356 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:35,356 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 22:11:35,356 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 7 +2025-01-21 22:11:35,356 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:11:35,357 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:35,357 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 22:11:35,357 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:35,357 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 22:11:35,422 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:11:35,460 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 8 +2025-01-21 22:11:35,461 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:11:35,461 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:35,461 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 22:11:35,461 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:35,461 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 22:11:35,461 INFO SenderThread:2661581 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 22:11:35,464 INFO SenderThread:2661581 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 22:11:35,466 INFO SenderThread:2661581 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 22:11:35,474 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 9 +2025-01-21 22:11:35,474 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:35,474 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 22:11:35,474 DEBUG SenderThread:2661581 [sender.py:send():379] send: artifact +2025-01-21 22:11:35,684 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:11:35,685 INFO Thread-12 :2661581 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:11:35,776 INFO SenderThread:2661581 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 22:11:35,776 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:35,776 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 22:11:35,776 INFO SenderThread:2661581 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 22:11:36,423 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:11:36,685 INFO SenderThread:2661581 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files +2025-01-21 22:11:36,685 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/config.yaml config.yaml +2025-01-21 22:11:36,685 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json wandb-summary.json +2025-01-21 22:11:36,686 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/requirements.txt requirements.txt +2025-01-21 22:11:36,686 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-metadata.json wandb-metadata.json +2025-01-21 22:11:36,686 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:11:36,690 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log output.log +2025-01-21 22:11:36,690 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/diff.patch diff.patch +2025-01-21 22:11:36,690 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 media/videos/eval/1_eval_1_27250e24fb2dabe74cd6.mp4 +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 media/videos/eval/0_eval_0_51125b9dabf4e9b3cedc.mp4 +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 media/videos/eval/4_eval_4_6afd64f2a3f512f6d9a6.mp4 +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 media/videos/eval/3_eval_3_3cfd6f1db2c9dcc9eb14.mp4 +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 media/videos/eval/2_eval_2_96f346beb19c014ed98c.mp4 +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json media/table/eval/total_success_rate_ci_6_41c18ac1da7a95c048c4.table.json +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-21 22:11:36,691 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 10 +2025-01-21 22:11:36,691 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:11:36,691 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:36,692 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 22:11:36,692 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:36,692 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 22:11:36,692 INFO SenderThread:2661581 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:11:36,881 INFO wandb-upload_0:2661581 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/wandb-summary.json +2025-01-21 22:11:36,918 INFO wandb-upload_2:2661581 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/requirements.txt +2025-01-21 22:11:36,936 INFO wandb-upload_1:2661581 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/config.yaml +2025-01-21 22:11:36,940 INFO wandb-upload_4:2661581 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/files/output.log +2025-01-21 22:11:37,007 INFO wandb-upload_3:2661581 [upload_job.py:push():130] Uploaded file /tmp/tmp_d_ntvepwandb/k5suzsul-events.out.tfevents.1737515229.leonmkim-ROG-Strix-G15CS-G15CS.2661386.0 +2025-01-21 22:11:37,207 INFO Thread-11 (_thread_body):2661581 [sender.py:transition_state():614] send defer: 11 +2025-01-21 22:11:37,207 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:37,208 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 22:11:37,208 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:37,208 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 22:11:37,208 INFO SenderThread:2661581 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 22:11:37,211 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 12 +2025-01-21 22:11:37,211 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:37,211 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 22:11:37,211 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:37,212 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 22:11:37,212 INFO SenderThread:2661581 [file_stream.py:finish():601] file stream finish called +2025-01-21 22:11:37,423 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:11:37,429 INFO SenderThread:2661581 [file_stream.py:finish():605] file stream finish is done +2025-01-21 22:11:37,429 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 13 +2025-01-21 22:11:37,429 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:11:37,429 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:37,429 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 22:11:37,429 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:37,430 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 22:11:37,430 INFO SenderThread:2661581 [sender.py:transition_state():614] send defer: 14 +2025-01-21 22:11:37,430 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:11:37,430 INFO HandlerThread:2661581 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 22:11:37,430 DEBUG SenderThread:2661581 [sender.py:send():379] send: final +2025-01-21 22:11:37,430 DEBUG SenderThread:2661581 [sender.py:send():379] send: footer +2025-01-21 22:11:37,430 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: defer +2025-01-21 22:11:37,430 INFO SenderThread:2661581 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 22:11:37,430 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:11:37,431 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 22:11:37,431 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:11:37,431 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:11:37,431 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: server_info +2025-01-21 22:11:37,433 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 22:11:37,433 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 22:11:37,434 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:11:37,485 DEBUG SenderThread:2661581 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:11:37,486 INFO MainThread:2661581 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 22:11:37,486 INFO MainThread:2661581 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 22:11:37,486 INFO MainThread:2661581 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 22:11:37,487 DEBUG HandlerThread:2661581 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 22:11:37,487 INFO HandlerThread:2661581 [handler.py:finish():882] shutting down handler +2025-01-21 22:11:38,433 INFO WriterThread:2661581 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/aqyye28j/wandb/run-20250121_220708-aqyye28j/run-aqyye28j.wandb +2025-01-21 22:11:38,486 INFO SenderThread:2661581 [sender.py:finish():1615] shutting down sender +2025-01-21 22:11:38,486 INFO SenderThread:2661581 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:11:38,486 INFO SenderThread:2661581 [file_pusher.py:join():175] waiting for file pusher diff --git a/m9j39xya/episode_rosbags/episode_0_2025-01-21-22-13-17.bag b/m9j39xya/episode_rosbags/episode_0_2025-01-21-22-13-17.bag new file mode 100644 index 0000000000000000000000000000000000000000..264bfc8a34ac3be59d60d994e36495c09ae7ff96 --- /dev/null +++ b/m9j39xya/episode_rosbags/episode_0_2025-01-21-22-13-17.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa6ab3216af636d15f51ed137adf423ecf027070abc5fa368df96250513abea2 +size 1057689154 diff --git a/m9j39xya/episode_rosbags/episode_1_2025-01-21-22-13-43.bag b/m9j39xya/episode_rosbags/episode_1_2025-01-21-22-13-43.bag new file mode 100644 index 0000000000000000000000000000000000000000..03c51d2803127575dbc3f89b405368db0e8cbddd --- /dev/null +++ b/m9j39xya/episode_rosbags/episode_1_2025-01-21-22-13-43.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f21e12a59e32b574b7233cbbe9c2c02a55e63a0b79a1cbdc4f0d330bdfa6071e +size 1039956457 diff --git a/m9j39xya/episode_rosbags/episode_2_2025-01-21-22-14-12.bag b/m9j39xya/episode_rosbags/episode_2_2025-01-21-22-14-12.bag new file mode 100644 index 0000000000000000000000000000000000000000..a23ca59911d12ff9f65d9649bc4bf725d7ba7976 --- /dev/null +++ b/m9j39xya/episode_rosbags/episode_2_2025-01-21-22-14-12.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8804306899b43a3356b8d17ccabba24f4e6ef987749060393ded25543026c546 +size 1338753310 diff --git a/uq9l7ajp/.hydra/config.yaml b/uq9l7ajp/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dc75d61dbe4b7ba0ede1ab77029649cc5b18502 --- /dev/null +++ b/uq9l7ajp/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/uq9l7ajp/.hydra/hydra.yaml b/uq9l7ajp/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59a3646767761b244cad12af7e4d13e6c20c1aef --- /dev/null +++ b/uq9l7ajp/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/uq9l7ajp + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false