diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..ded0c18e7e26952a34016875ed72ac46f351fc4e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,36 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 filter=lfs diff=lfs merge=lfs -text +zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 filter=lfs diff=lfs merge=lfs -text +zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 filter=lfs diff=lfs merge=lfs -text +zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 filter=lfs diff=lfs merge=lfs -text +zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 filter=lfs diff=lfs merge=lfs -text +zxwyz3xz/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +zxwyz3xz/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +zxwyz3xz/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +zxwyz3xz/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +zxwyz3xz/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 filter=lfs diff=lfs merge=lfs -text +4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 filter=lfs diff=lfs merge=lfs -text +4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 filter=lfs diff=lfs merge=lfs -text +4lg2e533/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +4lg2e533/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 filter=lfs diff=lfs merge=lfs -text +4lg2e533/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +4lg2e533/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 filter=lfs diff=lfs merge=lfs -text +4lg2e533/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +0e7f0awm/episode_rosbags/episode_0_2025-01-21-21-25-37.rrd filter=lfs diff=lfs merge=lfs -text +0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 filter=lfs diff=lfs merge=lfs -text +0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 filter=lfs diff=lfs merge=lfs -text +0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 filter=lfs diff=lfs merge=lfs -text +0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 filter=lfs diff=lfs merge=lfs -text +0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 filter=lfs diff=lfs merge=lfs -text +0e7f0awm/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +0e7f0awm/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +0e7f0awm/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +0e7f0awm/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +0e7f0awm/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nf7i3zuf/episode_rosbags/episode_1_2025-01-21-21-46-18.bag filter=lfs diff=lfs merge=lfs -text +nf7i3zuf/episode_rosbags/episode_3_2025-01-21-21-47-59.bag filter=lfs diff=lfs merge=lfs -text diff --git a/0e7f0awm/.hydra/config.yaml b/0e7f0awm/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2154a6acb64e2809a71d0adfc508bd71e0da7dde --- /dev/null +++ b/0e7f0awm/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/0e7f0awm/.hydra/hydra.yaml b/0e7f0awm/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb7c7822d7eb8f1430ba5f986afd880b5a0b87bc --- /dev/null +++ b/0e7f0awm/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/0e7f0awm/.hydra/overrides.yaml b/0e7f0awm/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/0e7f0awm/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/0e7f0awm/episode_rosbags/aligned_depth_to_color_K.npy b/0e7f0awm/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/0e7f0awm/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/0e7f0awm/episode_rosbags/cam_tf_world.npy b/0e7f0awm/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/0e7f0awm/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/0e7f0awm/episode_rosbags/color_K.npy b/0e7f0awm/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/0e7f0awm/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/0e7f0awm/episode_rosbags/depth_K.npy b/0e7f0awm/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/0e7f0awm/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/0e7f0awm/episode_rosbags/episode_0_2025-01-21-21-25-37.rrd b/0e7f0awm/episode_rosbags/episode_0_2025-01-21-21-25-37.rrd new file mode 100644 index 0000000000000000000000000000000000000000..7e38fe619496a3d9ec76939a2ef99f921b19a3e5 --- /dev/null +++ b/0e7f0awm/episode_rosbags/episode_0_2025-01-21-21-25-37.rrd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e72fab3d70c05b41b6188559b2186d2b32707b2f0078dfd62ced3c31cb47ed0e +size 272322799 diff --git a/0e7f0awm/eval_robot.log b/0e7f0awm/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..73633b581f78ba50a169971426b5df507b292e55 --- /dev/null +++ b/0e7f0awm/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 21:25:11,086][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:25:11,088][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/0e7f0awm/eval_video/0_eval.mp4 b/0e7f0awm/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5f88fc7edaf858dac49f3e13d4c18e6426ce2f59 --- /dev/null +++ b/0e7f0awm/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43ff0adeaa2aac56e705ffe46113ad0ea61aecc3b467f045c11c40cac3fc026e +size 1497303 diff --git a/0e7f0awm/eval_video/1_eval.mp4 b/0e7f0awm/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1ae1354cf77a3afefe073c060871b0575636a2b2 --- /dev/null +++ b/0e7f0awm/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b41bcf0751cacefe8bb11ec7af9ed5e5a069aa9e2900ac19b43d8ad540a4e4e +size 1533229 diff --git a/0e7f0awm/eval_video/2_eval.mp4 b/0e7f0awm/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..923531ddff6f93f8de59d5fe7d9c871ae4d05181 --- /dev/null +++ b/0e7f0awm/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebcc91651af67cca62de31104a2a79dbb6f62a076e08fa6d01ca662c3bdf77a2 +size 1542126 diff --git a/0e7f0awm/eval_video/3_eval.mp4 b/0e7f0awm/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..acd8c94b6ce3eb8abdb8ade96baa5d41e49acc06 --- /dev/null +++ b/0e7f0awm/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c7d164a44f31be2687d000530560b70a01353d005e918556e3a5e5ea9972a78 +size 1578241 diff --git a/0e7f0awm/eval_video/4_eval.mp4 b/0e7f0awm/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c5c118d25076f93bbc7a08d49bacf5557ca51c3 --- /dev/null +++ b/0e7f0awm/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dd664735f9b5c78730cb288e5b87cf1102ace7673ed83e702aab87668bca5ce +size 1549753 diff --git a/0e7f0awm/tb/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 b/0e7f0awm/tb/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 new file mode 100644 index 0000000000000000000000000000000000000000..69af9a0036fa31343a612a5117527a5d462917f8 --- /dev/null +++ b/0e7f0awm/tb/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32882a134dc757183a4815ff18174a23fe20c2892137a24cf79d686ceee1ca08 +size 6539 diff --git a/0e7f0awm/wandb/debug-internal.log b/0e7f0awm/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..145ca941bbccd9bd8f86ebf04e2643564f661c84 --- /dev/null +++ b/0e7f0awm/wandb/debug-internal.log @@ -0,0 +1,821 @@ +2025-01-21 21:25:16,840 INFO StreamThr :2650471 [internal.py:wandb_internal():85] W&B internal server running at pid: 2650471, started at: 2025-01-21 21:25:16.840076 +2025-01-21 21:25:16,841 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:25:16,854 INFO WriterThread:2650471 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/run-0e7f0awm.wandb +2025-01-21 21:25:16,855 DEBUG SenderThread:2650471 [sender.py:send():379] send: header +2025-01-21 21:25:16,862 DEBUG SenderThread:2650471 [sender.py:send():379] send: run +2025-01-21 21:25:17,186 INFO SenderThread:2650471 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files +2025-01-21 21:25:17,186 INFO SenderThread:2650471 [sender.py:_start_run_threads():1188] run started: 0e7f0awm with start time 1737512716.84077 +2025-01-21 21:25:17,201 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:25:17,201 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:25:17,280 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:25:17,285 DEBUG HandlerThread:2650471 [system_info.py:__init__():26] System info init +2025-01-21 21:25:17,285 DEBUG HandlerThread:2650471 [system_info.py:__init__():41] System info init done +2025-01-21 21:25:17,285 INFO HandlerThread:2650471 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:25:17,285 INFO SystemMonitor:2650471 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:25:17,285 INFO SystemMonitor:2650471 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:25:17,285 INFO HandlerThread:2650471 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:25:17,286 INFO SystemMonitor:2650471 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:25:17,286 INFO SystemMonitor:2650471 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:25:17,287 INFO SystemMonitor:2650471 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:25:17,288 INFO SystemMonitor:2650471 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:25:17,299 DEBUG HandlerThread:2650471 [system_info.py:probe():152] Probing system +2025-01-21 21:25:17,300 DEBUG HandlerThread:2650471 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:25:17,305 DEBUG HandlerThread:2650471 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:25:17,305 DEBUG HandlerThread:2650471 [system_info.py:probe():200] Probing system done +2025-01-21 21:25:17,305 DEBUG HandlerThread:2650471 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:25:17.299296', 'startedAt': '2025-01-22T02:25:16.828689', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.83275, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4491.548, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.283, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 369.33309173583984}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:25:17,305 INFO HandlerThread:2650471 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:25:17,305 INFO HandlerThread:2650471 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:25:17,305 DEBUG HandlerThread:2650471 [system_info.py:_save_code():44] Saving code +2025-01-21 21:25:17,307 DEBUG HandlerThread:2650471 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:25:17,307 DEBUG HandlerThread:2650471 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:25:17,368 DEBUG HandlerThread:2650471 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:25:17,369 INFO HandlerThread:2650471 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:25:17,372 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:25:17,372 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:25:17,372 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:25:17,372 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:25:17,621 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/8gubrilm-wandb-metadata.json +2025-01-21 21:25:17,628 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:25:17,629 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:25:17,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:25:17,629 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:25:17,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:17,631 INFO HandlerThread:2650471 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/tb" + save: true +} +control { + relay_id: "b0bacdb30a8a4a9e968f1340a11639e4" +} +_info { + stream_id: "0e7f0awm" +} + +2025-01-21 21:25:17,631 DEBUG HandlerThread:2650471 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:25:17,658 INFO wandb-upload_1:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/qnenjw8n-code/FISH/eval_robot.py +2025-01-21 21:25:17,680 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/hiqogtmz-diff.patch +2025-01-21 21:25:17,768 DEBUG SenderThread:2650471 [sender.py:send():379] send: telemetry +2025-01-21 21:25:17,768 DEBUG SenderThread:2650471 [sender.py:send():379] send: tbrecord +2025-01-21 21:25:17,769 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:25:17,769 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 with policy live +2025-01-21 21:25:17,961 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/nnvnqxl1-events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/code/FISH/eval_robot.py +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/diff.patch +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-metadata.json +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/requirements.txt +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/code/FISH +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/code +2025-01-21 21:25:18,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:19,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:20,187 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:20,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:21,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:22,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:25:22,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:23,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:24,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:24,844 DEBUG SenderThread:2650471 [sender.py:send():379] send: config +2025-01-21 21:25:25,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:26,189 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:26,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:27,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:25:27,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:28,189 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:28,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:29,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:30,190 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:30,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:31,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:32,191 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:32,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:25:32,629 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:25:32,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:32,786 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:25:33,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:34,191 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:34,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:35,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:36,192 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:36,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:37,633 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:38,193 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:38,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:25:38,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:39,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:40,193 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:40,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:41,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:42,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:43,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:25:43,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:44,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:45,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:46,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:47,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:25:47,629 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:25:47,677 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:48,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:48,768 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:25:49,196 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/config.yaml +2025-01-21 21:25:49,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:50,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:51,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:52,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:53,639 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:54,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:25:54,639 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:55,639 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:56,639 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:57,639 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:58,639 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:59,640 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:00,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:00,640 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:01,640 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:02,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:26:02,629 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:26:02,673 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:03,640 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:04,640 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:05,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:05,789 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:06,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:07,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:08,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:09,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:10,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:11,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:11,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:12,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:13,281 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:26:13,281 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 with policy now +2025-01-21 21:26:13,281 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:26:13,282 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:26:13,282 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:26:13,283 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:26:13,545 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/wkmrntx1-media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 +2025-01-21 21:26:13,642 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:14,202 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:26:14,203 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:26:14,203 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 +2025-01-21 21:26:14,204 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval +2025-01-21 21:26:14,204 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media +2025-01-21 21:26:14,204 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos +2025-01-21 21:26:14,512 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/cot1kqu8-media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 +2025-01-21 21:26:14,642 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:15,642 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:16,643 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:16,895 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:17,288 DEBUG SystemMonitor:2650471 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:26:17,289 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:26:17,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:26:17,629 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:26:17,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:18,204 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:26:18,642 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:19,645 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:20,643 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:21,643 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:22,205 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:26:22,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:22,643 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:23,643 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:24,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:25,645 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:26,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:27,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:27,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:28,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:29,208 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:26:29,487 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/tzl4cl22-events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:26:29,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:30,208 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:26:30,645 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:31,650 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:32,209 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:26:32,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:26:32,630 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:26:32,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:32,743 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:33,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:34,209 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:26:34,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:35,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:36,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:37,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:38,632 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:38,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:39,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:40,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:41,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:42,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:43,632 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:43,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:44,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:45,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:46,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:47,298 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:26:47,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:26:47,630 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:26:47,673 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:48,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:48,789 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:49,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:50,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:51,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:52,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:53,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:54,633 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:54,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:55,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:56,649 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:57,649 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:58,649 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:59,649 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:00,633 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:00,649 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:01,649 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:02,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:27:02,630 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:27:02,673 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:03,650 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:04,650 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:05,650 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:05,771 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:06,650 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:07,653 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:08,101 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:27:08,102 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 with policy now +2025-01-21 21:27:08,102 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:27:08,102 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:27:08,103 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:27:08,105 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:27:08,220 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:27:08,220 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:08,220 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 +2025-01-21 21:27:08,221 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval +2025-01-21 21:27:08,379 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/vzmwqlwa-media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 +2025-01-21 21:27:08,650 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/e7so9tju-media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 +2025-01-21 21:27:08,658 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:09,663 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:10,221 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:10,659 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:11,128 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:11,660 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:12,661 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:13,660 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:14,223 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:14,662 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:15,661 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:16,223 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:16,224 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:27:16,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:16,921 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:17,058 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:27:17,058 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:27:17,058 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:27:17,224 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:27:17,290 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:27:17,634 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:27:17,634 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:27:17,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:18,224 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:18,667 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:19,693 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:20,663 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:21,663 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:22,226 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:22,226 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:22,663 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:23,663 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:24,663 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:25,663 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:26,664 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:27,637 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:27,664 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:28,664 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:29,664 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:30,664 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:31,664 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:32,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:27:32,630 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:27:32,673 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:32,737 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:33,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:34,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:35,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:36,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:37,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:38,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:38,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:39,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:40,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:41,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:42,232 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:42,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:43,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:44,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:44,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:45,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:46,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:47,290 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:27:47,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:27:47,630 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:27:47,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:48,234 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:48,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:49,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:49,724 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:50,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:51,667 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:52,667 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:53,667 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:54,236 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:54,667 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:55,599 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:55,667 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:56,607 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:27:56,607 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 with policy now +2025-01-21 21:27:56,609 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:27:56,610 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:27:56,610 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:27:56,614 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:27:56,677 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:56,871 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/6y9lw6pv-media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 +2025-01-21 21:27:57,237 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:27:57,237 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 +2025-01-21 21:27:57,238 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval +2025-01-21 21:27:57,536 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/8b6uehix-media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 +2025-01-21 21:27:57,670 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:58,238 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:58,672 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:59,673 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:00,673 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:01,297 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:01,678 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:02,239 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:02,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:28:02,635 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:28:02,677 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:03,676 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:04,240 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:04,674 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:05,676 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:06,677 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:07,090 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:07,678 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:08,241 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:28:08,503 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/x7ibo417-events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:28:08,675 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:08,898 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:28:08,898 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:28:08,898 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:28:09,241 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:28:09,241 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:28:09,680 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:10,242 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:10,679 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:11,682 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:12,117 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:12,682 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:13,680 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:14,243 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:14,680 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:15,680 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:16,244 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:16,680 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:17,291 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:28:17,291 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:17,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:28:17,631 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:28:17,680 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:18,680 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:19,681 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:20,681 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:21,681 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:22,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:22,681 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:23,681 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:24,681 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:25,681 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:26,682 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:27,682 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:28,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:28,682 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:29,682 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:30,683 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:31,683 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:32,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:28:32,631 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:28:32,683 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:33,683 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:33,754 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:34,683 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:35,685 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:36,684 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:37,684 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:38,690 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:39,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:39,688 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:40,688 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:41,689 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:42,733 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:43,692 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:44,253 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:44,691 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:44,734 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:45,691 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:46,691 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:47,291 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:28:47,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:28:47,631 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:28:47,691 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:48,691 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:49,707 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:49,805 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:50,124 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:28:50,125 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 with policy now +2025-01-21 21:28:50,126 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:28:50,128 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:28:50,128 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:28:50,139 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:28:50,255 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:28:50,255 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:50,255 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 +2025-01-21 21:28:50,256 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval +2025-01-21 21:28:50,402 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/pq6izpce-media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 +2025-01-21 21:28:50,706 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:50,706 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/jxcyo4ia-media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 +2025-01-21 21:28:51,705 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:52,256 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:52,704 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:53,706 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:54,708 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:55,707 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:55,805 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:56,257 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:56,706 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:57,713 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:58,258 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:58,713 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:59,714 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:00,714 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:00,906 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:00,959 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:29:00,959 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:29:00,960 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:29:01,259 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:29:01,259 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:29:01,710 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:02,259 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:02,633 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:29:02,634 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:29:02,717 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:03,726 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:04,260 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:04,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:05,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:06,261 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:06,307 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:06,722 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:07,717 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:08,262 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:08,717 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:09,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:10,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:11,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:12,025 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:12,263 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:12,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:13,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:14,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:15,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:16,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:17,291 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:29:17,293 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:17,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:29:17,632 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:29:17,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:18,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:19,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:20,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:21,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:22,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:22,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:23,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:24,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:25,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:26,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:27,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:27,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:28,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:29,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:30,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:31,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:32,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:29:32,631 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:29:32,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:32,775 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:33,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:34,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:35,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:36,723 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:37,721 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:38,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:38,721 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:39,721 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:40,721 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:41,739 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:42,249 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:29:42,249 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 with policy now +2025-01-21 21:29:42,250 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:29:42,250 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:29:42,251 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:29:42,277 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:29:42,277 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:42,277 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 +2025-01-21 21:29:42,277 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval +2025-01-21 21:29:42,277 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:29:42,536 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/r0tc60x5-media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 +2025-01-21 21:29:42,737 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:42,797 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/x2euuyc5-media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 +2025-01-21 21:29:43,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:43,741 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:44,277 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:44,743 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:45,741 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:46,746 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:47,292 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:29:47,642 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:29:47,642 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:29:47,754 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:48,279 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:48,708 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:48,757 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:49,756 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:50,280 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:50,754 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:51,759 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:52,759 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:53,758 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:54,659 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:54,759 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:55,768 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:56,767 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:57,765 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:58,764 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:59,776 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:00,056 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:30:00,056 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:00,056 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:00,057 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:00,283 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:30:00,283 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:30:00,283 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:30:00,527 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:30:00,527 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:30:00,554 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/pvin124k-events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:30:00,686 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:30:00,763 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:01,049 INFO wandb-upload_0:2650471 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpg6369ocs +2025-01-21 21:30:01,070 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:30:01,086 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:30:01,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:30:01,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-21 21:30:01,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 21:30:01,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media +2025-01-21 21:30:01,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table +2025-01-21 21:30:01,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval +2025-01-21 21:30:01,614 INFO SenderThread:2650471 [sender.py:send_request_log_artifact():1525] logged artifact run-0e7f0awm-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3Mjk4Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5NjQ1MQ==', 'latestArtifact': None}} +2025-01-21 21:30:01,614 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:30:01,614 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-21 21:30:01,614 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:30:01,614 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,615 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,615 DEBUG SenderThread:2650471 [sender.py:send():379] send: telemetry +2025-01-21 21:30:01,615 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,616 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,616 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,616 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,616 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,616 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,617 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,617 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,617 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,617 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,618 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,618 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,618 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,618 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,618 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,619 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,619 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,619 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,619 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,620 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,620 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,620 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,620 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:30:01,763 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:01,803 INFO wandb-upload_1:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/vahap6lh-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 21:30:01,852 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/ciug7oaw-media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-21 21:30:02,040 INFO wandb-upload_0:2650471 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzvvzrdlq +2025-01-21 21:30:02,091 INFO wandb-upload_1:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/hyogeujo-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 21:30:02,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:30:02,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:30:02,462 INFO SenderThread:2650471 [sender.py:send_request_log_artifact():1525] logged artifact run-0e7f0awm-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3Mjk5OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5NjQ1NA==', 'latestArtifact': None}} +2025-01-21 21:30:02,462 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:30:02,462 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json with policy now +2025-01-21 21:30:02,462 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:30:02,462 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:02,463 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:02,632 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:30:02,632 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:30:02,740 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/myjeghn4-media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-21 21:30:02,765 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:03,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:30:03,764 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:04,285 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:30:04,764 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:05,698 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:05,764 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:06,764 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:07,764 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:08,765 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:09,765 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:10,765 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:11,661 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:11,780 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:12,288 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/config.yaml +2025-01-21 21:30:12,766 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:13,765 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:14,765 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:15,766 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:16,766 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:17,292 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:30:17,293 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:17,633 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:30:17,633 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:30:17,766 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:18,165 DEBUG SenderThread:2650471 [sender.py:send():379] send: exit +2025-01-21 21:30:18,166 INFO SenderThread:2650471 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:30:18,166 INFO SenderThread:2650471 [sender.py:send_exit():588] handling runtime: 300 +2025-01-21 21:30:18,166 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:18,166 INFO SenderThread:2650471 [sender.py:send_exit():594] send defer +2025-01-21 21:30:18,166 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:18,166 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:30:18,166 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:18,166 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:30:18,167 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:30:18,167 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:18,167 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:30:18,167 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:18,167 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:30:18,167 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:30:18,167 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:18,167 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:30:18,167 INFO HandlerThread:2650471 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:30:18,167 INFO HandlerThread:2650471 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:30:18,168 DEBUG SystemMonitor:2650471 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:30:18,168 INFO HandlerThread:2650471 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:30:18,168 DEBUG SystemMonitor:2650471 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:30:18,222 INFO HandlerThread:2650471 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:30:18,222 INFO HandlerThread:2650471 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:30:18,222 INFO HandlerThread:2650471 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:30:18,223 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:18,223 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:30:18,223 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:30:18,223 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:30:18,223 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:18,223 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:30:18,223 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:18,223 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:30:18,223 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:30:18,223 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:18,223 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:30:18,290 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:30:20,291 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:30:25,063 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:30:25,063 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:25,063 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:25,064 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:30:25,064 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:30:25,064 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:30:25,065 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:30:25,065 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:25,065 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:30:25,066 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:30:25,066 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:25,066 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:25,066 DEBUG SenderThread:2650471 [sender.py:send():379] send: summary +2025-01-21 21:30:25,067 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:25,067 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:25,067 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:30:25,067 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:30:25,067 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:25,067 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:30:25,067 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:25,067 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:30:25,068 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:30:25,068 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:25,068 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:25,068 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:30:25,068 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:25,068 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:30:25,167 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:30:25,293 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:30:26,085 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:30:26,085 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:30:26,085 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:26,085 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:30:26,086 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:26,086 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:30:26,086 INFO SenderThread:2650471 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:30:26,088 INFO SenderThread:2650471 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:30:26,093 INFO SenderThread:2650471 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:30:26,107 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:30:26,108 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:26,108 DEBUG SenderThread:2650471 [sender.py:send():379] send: artifact +2025-01-21 21:30:26,108 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:30:26,167 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:30:26,294 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:30:26,513 INFO wandb-upload_0:2650471 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgla44l70 +2025-01-21 21:30:26,609 INFO wandb-upload_1:2650471 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzfmbo_92 +2025-01-21 21:30:26,660 INFO wandb-upload_2:2650471 [upload_job.py:push():88] Uploaded file /tmp/tmp8vwc9knz/wandb-job.json +2025-01-21 21:30:27,273 INFO SenderThread:2650471 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3MzQyMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'versionIndex': 454}}} +2025-01-21 21:30:27,273 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:27,273 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:30:27,273 INFO SenderThread:2650471 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:30:27,295 INFO SenderThread:2650471 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files +2025-01-21 21:30:27,295 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/config.yaml config.yaml +2025-01-21 21:30:27,295 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json wandb-summary.json +2025-01-21 21:30:27,295 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/requirements.txt requirements.txt +2025-01-21 21:30:27,295 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:30:27,295 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:30:27,295 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log output.log +2025-01-21 21:30:27,296 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/diff.patch diff.patch +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 21:30:27,302 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:30:27,302 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:30:27,302 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:27,302 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:30:27,302 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:27,304 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:30:27,305 INFO SenderThread:2650471 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:30:27,480 INFO wandb-upload_1:2650471 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:30:27,486 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/requirements.txt +2025-01-21 21:30:27,507 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/config.yaml +2025-01-21 21:30:27,586 INFO wandb-upload_4:2650471 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:30:27,641 INFO wandb-upload_3:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/rm9m3ufd-events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:30:27,842 INFO Thread-11 (_thread_body):2650471 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:30:27,842 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:27,842 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:30:27,842 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:27,842 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:30:27,842 INFO SenderThread:2650471 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:30:27,844 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:30:27,844 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:27,844 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:30:27,844 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:27,844 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:30:27,844 INFO SenderThread:2650471 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:30:28,069 INFO SenderThread:2650471 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:30:28,069 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:30:28,069 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:28,069 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:30:28,069 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:28,069 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:30:28,069 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:30:28,069 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:28,069 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:30:28,070 DEBUG SenderThread:2650471 [sender.py:send():379] send: final +2025-01-21 21:30:28,070 DEBUG SenderThread:2650471 [sender.py:send():379] send: footer +2025-01-21 21:30:28,070 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:28,070 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:30:28,070 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:30:28,070 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:30:28,071 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:30:28,071 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:30:28,072 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:30:28,072 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:30:28,073 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:30:28,073 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:28,126 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:30:28,126 INFO MainThread:2650471 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:30:28,126 INFO MainThread:2650471 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:30:28,127 INFO MainThread:2650471 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:30:28,127 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:30:28,127 INFO HandlerThread:2650471 [handler.py:finish():882] shutting down handler +2025-01-21 21:30:29,072 INFO WriterThread:2650471 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/run-0e7f0awm.wandb +2025-01-21 21:30:29,126 INFO SenderThread:2650471 [sender.py:finish():1615] shutting down sender +2025-01-21 21:30:29,126 INFO SenderThread:2650471 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:30:29,126 INFO SenderThread:2650471 [file_pusher.py:join():175] waiting for file pusher diff --git a/0e7f0awm/wandb/debug.log b/0e7f0awm/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8b7f073f1bafbce91679ebeb90493507fbe48233 --- /dev/null +++ b/0e7f0awm/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Configure stats pid to 2650272 +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/settings +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/logs/debug.log +2025-01-21 21:25:16,835 INFO MainThread:2650272 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/logs/debug-internal.log +2025-01-21 21:25:16,835 INFO MainThread:2650272 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:25:16,835 INFO MainThread:2650272 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_1', 'wandb_notes': '4295_1_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm'} +2025-01-21 21:25:16,835 INFO MainThread:2650272 [wandb_init.py:init():619] starting backend +2025-01-21 21:25:16,835 INFO MainThread:2650272 [wandb_init.py:init():623] setting up manager +2025-01-21 21:25:16,839 INFO MainThread:2650272 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:25:16,840 INFO MainThread:2650272 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:25:16,854 INFO MainThread:2650272 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:25:16,860 INFO MainThread:2650272 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:25:17,200 INFO MainThread:2650272 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:25:17,274 INFO MainThread:2650272 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:25:17,275 INFO MainThread:2650272 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:25:17,629 INFO MainThread:2650272 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:25:17,629 INFO MainThread:2650272 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:25:17,629 INFO MainThread:2650272 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:25:17,629 INFO MainThread:2650272 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:25:17,631 INFO MainThread:2650272 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:25:17,631 INFO MainThread:2650272 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/tb, True +2025-01-21 21:25:24,844 INFO MainThread:2650272 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:30:29,128 WARNING MsgRouterThr:2650272 [router.py:message_loop():77] message_loop has been closed diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/code/FISH/eval_robot.py b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/config.yaml b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9dfc3c2767b5afd01cac324ee35924457130f2df --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_1' +wandb_notes: + desc: null + value: 4295_1_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737512716 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/diff.patch b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..0d1d1c2d34392be0466354e2f5a3da5c22dfe3f1 --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/diff.patch @@ -0,0 +1,24 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..3a697a3 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False +@@ -144,8 +144,8 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '3465_2' # seed 2 + + # crp_D+mask+contact+act history +-wandb_run_id: '4295_0' +-# wandb_run_id: '4295_1' # seed 1 ++# wandb_run_id: '4295_0' ++wandb_run_id: '4295_1' # seed 1 + # wandb_run_id: '4295_2' # seed 2 + + # RGBD+mask+act history diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0cf4c88c517af93d38ad4b5d570fa02efc2afe50 --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[5, 5, 1.0, 0.9786591528888874, 1, 0.5107108652570282, 1, 0.5613806098576808, 1, 1, 1]]} \ No newline at end of file diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5f88fc7edaf858dac49f3e13d4c18e6426ce2f59 --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43ff0adeaa2aac56e705ffe46113ad0ea61aecc3b467f045c11c40cac3fc026e +size 1497303 diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1ae1354cf77a3afefe073c060871b0575636a2b2 --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b41bcf0751cacefe8bb11ec7af9ed5e5a069aa9e2900ac19b43d8ad540a4e4e +size 1533229 diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..923531ddff6f93f8de59d5fe7d9c871ae4d05181 --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebcc91651af67cca62de31104a2a79dbb6f62a076e08fa6d01ca662c3bdf77a2 +size 1542126 diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..acd8c94b6ce3eb8abdb8ade96baa5d41e49acc06 --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c7d164a44f31be2687d000530560b70a01353d005e918556e3a5e5ea9972a78 +size 1578241 diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c5c118d25076f93bbc7a08d49bacf5557ca51c3 --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dd664735f9b5c78730cb288e5b87cf1102ace7673ed83e702aab87668bca5ce +size 1549753 diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b8b6a74aa077ccfe4314b55f3ae85e5c2f0ab1f8 --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log @@ -0,0 +1,264 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737512724.845518]: resetting environment +[INFO] [1737512724.852802]: cleared current plan +[INFO] [1737512724.853707]: moving to home +[INFO] [1737512725.855743]: reached home +[INFO] [1737512725.857075]: reset action history +[INFO] [1737512728.567875]: environment reset +Starting episode 0 +[INFO] [1737512728.572582]: resetting environment +[INFO] [1737512728.577631]: cleared current plan +[INFO] [1737512728.578729]: moving to home +[INFO] [1737512729.581264]: reached home +[INFO] [1737512729.582628]: reset action history +[INFO] [1737512732.292210]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737512733.564496]: resetting environment +[INFO] [1737512733.568739]: cleared current plan +[INFO] [1737512733.569833]: moving to home +[INFO] [1737512734.572695]: reached home +[INFO] [1737512734.573899]: reset action history +[INFO] [1737512737.282510]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/episode_rosbags/episode_0_2025-01-21-21-25-37.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2650272.log +started roslaunch server http://158.130.50.37:44097/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2650577] +process[print_text-2]: started with pid [2650601] +process[rosbag_record-3]: started with pid [2650602] +started bagging! +[INFO] [1737512772.293201]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737512773.050043]: Stored episode 1. +[INFO] [1737512773.055691]: Saving video... +[INFO] [1737512773.263689]: Video saved! +Starting episode 1 +[INFO] [1737512773.282404]: resetting environment +[INFO] [1737512773.291131]: cleared current plan +[INFO] [1737512773.291401]: moving to home +[INFO] [1737512776.894495]: reached home +[INFO] [1737512776.894943]: reset action history +[INFO] [1737512779.663930]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737512788.443367]: resetting environment +[INFO] [1737512788.447132]: cleared current plan +[INFO] [1737512788.447319]: moving to home +[INFO] [1737512789.448732]: reached home +[INFO] [1737512789.449382]: reset action history +[INFO] [1737512792.159430]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/episode_rosbags/episode_1_2025-01-21-21-26-32.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2650272.log +started roslaunch server http://158.130.50.37:42095/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2650756] +process[print_text-5]: started with pid [2650757] +process[rosbag_record-6]: started with pid [2650758] +started bagging! +[INFO] [1737512827.081540]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1737512827.833222]: Stored episode 2. +[INFO] [1737512827.845796]: Saving video... +[INFO] [1737512828.072531]: Video saved! +Starting episode 2 +[INFO] [1737512828.103480]: resetting environment +[INFO] [1737512828.123565]: cleared current plan +[INFO] [1737512828.127642]: moving to home +[INFO] [1737512831.744768]: reached home +[INFO] [1737512831.749003]: reset action history +[INFO] [1737512834.472909]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737512835.894581]: resetting environment +[INFO] [1737512835.912276]: cleared current plan +[INFO] [1737512835.915297]: moving to home +[INFO] [1737512836.918894]: reached home +[INFO] [1737512836.919698]: reset action history +[INFO] [1737512839.646399]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/episode_rosbags/episode_2_2025-01-21-21-27-19.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2650272.log +started roslaunch server http://158.130.50.37:37843/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2650900] +process[print_text-8]: started with pid [2650908] +process[rosbag_record-9]: started with pid [2650909] +started bagging! +For topic gripper_width: timestamp difference is 100456463 for nearest: 1737512861079159723 - target: 1737512860978703260 at idx 249 +For topic gripper_width: timestamp difference is 112619277 for nearest: 1737512867079105254 - target: 1737512866966485977 at idx 249 +For topic gripper_width: timestamp difference is 111569451 for nearest: 1737512871979117440 - target: 1737512871867547989 at idx 249 +[INFO] [1737512875.598048]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1737512876.353447]: Stored episode 3. +[INFO] [1737512876.356951]: Saving video... +[INFO] [1737512876.573018]: Video saved! +Starting episode 3 +[INFO] [1737512876.616644]: resetting environment +[INFO] [1737512876.644454]: cleared current plan +[INFO] [1737512876.647381]: moving to home +[INFO] [1737512880.252689]: reached home +[INFO] [1737512880.261163]: reset action history +[INFO] [1737512883.047823]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737512888.268791]: resetting environment +[INFO] [1737512888.286587]: cleared current plan +[INFO] [1737512888.286812]: moving to home +[INFO] [1737512889.288758]: reached home +[INFO] [1737512889.293517]: reset action history +[INFO] [1737512892.070584]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/episode_rosbags/episode_3_2025-01-21-21-28-12.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2650272.log +started roslaunch server http://158.130.50.37:45317/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2651042] +process[print_text-11]: started with pid [2651043] +process[rosbag_record-12]: started with pid [2651044] +started bagging! +For topic gripper_width: timestamp difference is 117910521 for nearest: 1737512922679159300 - target: 1737512922561248779 at idx 249 +[INFO] [1737512928.848335]: Storing episode... +[pub_text-10] killing on exit[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[INFO] [1737512929.804549]: Stored episode 4. +[INFO] [1737512929.818505]: Saving video... +[INFO] [1737512930.077425]: Video saved! +Starting episode 4 +[INFO] [1737512930.131616]: resetting environment +[INFO] [1737512930.151739]: cleared current plan +[INFO] [1737512930.152471]: moving to home +[INFO] [1737512933.761069]: reached home +[INFO] [1737512933.763967]: reset action history +[INFO] [1737512936.551983]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737512940.947392]: resetting environment +[INFO] [1737512940.993072]: cleared current plan +[INFO] [1737512940.996505]: moving to home +[INFO] [1737512942.003433]: reached home +[INFO] [1737512942.006438]: reset action history +[INFO] [1737512944.797894]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/episode_rosbags/episode_4_2025-01-21-21-29-04.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2650272.log +started roslaunch server http://158.130.50.37:45483/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2651182] +process[print_text-14]: started with pid [2651183] +process[rosbag_record-15]: started with pid [2651184] +started bagging! +For topic gripper_width: timestamp difference is 102262214 for nearest: 1737512949979182676 - target: 1737512949876920462 at idx 67 +[INFO] [1737512981.086152]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1737512981.952518]: Stored episode 5. +[INFO] [1737512981.954539]: Saving video... +[INFO] [1737512982.180001]: Video saved! +[INFO] [1737512982.250940]: resetting environment +[INFO] [1737512982.289704]: cleared current plan +[INFO] [1737512982.290060]: moving to home +[INFO] [1737512985.893738]: reached home +[INFO] [1737512985.900269]: reset action history +[INFO] [1737512988.707855]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/requirements.txt b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-metadata.json b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..181deb081158d1b1c1773045bc8b4e7b3bb963f7 --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T02:25:17.299296", + "startedAt": "2025-01-22T02:25:16.828689", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3561.83275, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4491.548, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.283, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 369.33309173583984 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..972f185f4ac5d45919934342714e20b57b7a03ef --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "43ff0adeaa2aac56e705ffe46113ad0ea61aecc3b467f045c11c40cac3fc026e", "size": 1497303, "path": "media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4"}, "global_step": 665, "_timestamp": 1737513000.961645, "_runtime": 284.1208748817444, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "6b41bcf0751cacefe8bb11ec7af9ed5e5a069aa9e2900ac19b43d8ad540a4e4e", "size": 1533229, "path": "media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4"}, "eval/num_success": 5.0, "episode": 5.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "ebcc91651af67cca62de31104a2a79dbb6f62a076e08fa6d01ca662c3bdf77a2", "size": 1542126, "path": "media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "6c7d164a44f31be2687d000530560b70a01353d005e918556e3a5e5ea9972a78", "size": 1578241, "path": "media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "1dd664735f9b5c78730cb288e5b87cf1102ace7673ed83e702aab87668bca5ce", "size": 1549753, "path": "media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://mc2paq05dcpv4j4f688zjbcbmr2a5t4x5ob87vshdkng1xtu5z1yw9pktfsq5l4vt97mlv1i1b8ezxufam8q44ng3n7ftsk5o71js1cbd7y4rsvjwxmdfg6ny9dt8ohl/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://uu1ew5o7c9bxm23msrif9wc6isozujoi80eywp6y3wwhblwbn6un3ho5wdi0xyx4gh5ph6bm2ahyr74vnz4mgt22yurj875oeydin4si2p1ptiucqik4mp100i9o7zrz:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 5, "total_num_episodes": 5, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9786591528888874, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.5107108652570282, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.5613806098576808, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "e3405f8336f17a4a59dd9de56afebbdceab20a8c5c8ce9c3ea21635b9f54eaac", "size": 418, "artifact_path": "wandb-client-artifact://s88w972zuhay530jxnggfwha7krxsehf10rmueb4qcz95j3764rjb5h7p2fa4nhxxb619d8xomqhcxfnytfcqo98meultjfyeuzo9my5vi4la3mo4nv4xgz5dup4tyz7/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://deg9a3jj150npt100r5jnkug0ot4b923pbxquqi13cw47hvijaols8kv2njyt3qfkfc8m2ahrwqkq74a7eqzccssxni5lwewbq6vu95asmktm1ml2nkzqfpagf42ocza:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 300}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9786591529846191, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.5107108354568481, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.5613806247711182, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/logs/debug-internal.log b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..145ca941bbccd9bd8f86ebf04e2643564f661c84 --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/logs/debug-internal.log @@ -0,0 +1,821 @@ +2025-01-21 21:25:16,840 INFO StreamThr :2650471 [internal.py:wandb_internal():85] W&B internal server running at pid: 2650471, started at: 2025-01-21 21:25:16.840076 +2025-01-21 21:25:16,841 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:25:16,854 INFO WriterThread:2650471 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/run-0e7f0awm.wandb +2025-01-21 21:25:16,855 DEBUG SenderThread:2650471 [sender.py:send():379] send: header +2025-01-21 21:25:16,862 DEBUG SenderThread:2650471 [sender.py:send():379] send: run +2025-01-21 21:25:17,186 INFO SenderThread:2650471 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files +2025-01-21 21:25:17,186 INFO SenderThread:2650471 [sender.py:_start_run_threads():1188] run started: 0e7f0awm with start time 1737512716.84077 +2025-01-21 21:25:17,201 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:25:17,201 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:25:17,280 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:25:17,285 DEBUG HandlerThread:2650471 [system_info.py:__init__():26] System info init +2025-01-21 21:25:17,285 DEBUG HandlerThread:2650471 [system_info.py:__init__():41] System info init done +2025-01-21 21:25:17,285 INFO HandlerThread:2650471 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:25:17,285 INFO SystemMonitor:2650471 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:25:17,285 INFO SystemMonitor:2650471 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:25:17,285 INFO HandlerThread:2650471 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:25:17,286 INFO SystemMonitor:2650471 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:25:17,286 INFO SystemMonitor:2650471 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:25:17,287 INFO SystemMonitor:2650471 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:25:17,288 INFO SystemMonitor:2650471 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:25:17,299 DEBUG HandlerThread:2650471 [system_info.py:probe():152] Probing system +2025-01-21 21:25:17,300 DEBUG HandlerThread:2650471 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:25:17,305 DEBUG HandlerThread:2650471 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:25:17,305 DEBUG HandlerThread:2650471 [system_info.py:probe():200] Probing system done +2025-01-21 21:25:17,305 DEBUG HandlerThread:2650471 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:25:17.299296', 'startedAt': '2025-01-22T02:25:16.828689', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.83275, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4491.548, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.283, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 369.33309173583984}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:25:17,305 INFO HandlerThread:2650471 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:25:17,305 INFO HandlerThread:2650471 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:25:17,305 DEBUG HandlerThread:2650471 [system_info.py:_save_code():44] Saving code +2025-01-21 21:25:17,307 DEBUG HandlerThread:2650471 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:25:17,307 DEBUG HandlerThread:2650471 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:25:17,368 DEBUG HandlerThread:2650471 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:25:17,369 INFO HandlerThread:2650471 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:25:17,372 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:25:17,372 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:25:17,372 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:25:17,372 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:25:17,621 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/8gubrilm-wandb-metadata.json +2025-01-21 21:25:17,628 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:25:17,629 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:25:17,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:25:17,629 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:25:17,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:17,631 INFO HandlerThread:2650471 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/tb" + save: true +} +control { + relay_id: "b0bacdb30a8a4a9e968f1340a11639e4" +} +_info { + stream_id: "0e7f0awm" +} + +2025-01-21 21:25:17,631 DEBUG HandlerThread:2650471 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:25:17,658 INFO wandb-upload_1:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/qnenjw8n-code/FISH/eval_robot.py +2025-01-21 21:25:17,680 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/hiqogtmz-diff.patch +2025-01-21 21:25:17,768 DEBUG SenderThread:2650471 [sender.py:send():379] send: telemetry +2025-01-21 21:25:17,768 DEBUG SenderThread:2650471 [sender.py:send():379] send: tbrecord +2025-01-21 21:25:17,769 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:25:17,769 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 with policy live +2025-01-21 21:25:17,961 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/nnvnqxl1-events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/code/FISH/eval_robot.py +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/diff.patch +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-metadata.json +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/requirements.txt +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/code/FISH +2025-01-21 21:25:18,188 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/code +2025-01-21 21:25:18,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:19,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:20,187 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:20,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:21,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:22,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:25:22,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:23,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:24,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:24,844 DEBUG SenderThread:2650471 [sender.py:send():379] send: config +2025-01-21 21:25:25,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:26,189 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:26,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:27,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:25:27,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:28,189 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:28,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:29,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:30,190 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:30,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:31,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:32,191 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:32,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:25:32,629 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:25:32,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:32,786 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:25:33,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:34,191 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:34,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:35,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:36,192 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:36,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:37,633 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:38,193 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:38,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:25:38,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:39,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:40,193 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:25:40,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:41,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:42,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:43,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:25:43,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:44,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:45,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:46,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:47,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:25:47,629 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:25:47,677 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:48,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:48,768 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:25:49,196 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/config.yaml +2025-01-21 21:25:49,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:50,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:51,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:52,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:53,639 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:54,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:25:54,639 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:55,639 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:56,639 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:57,639 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:58,639 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:25:59,640 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:00,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:00,640 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:01,640 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:02,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:26:02,629 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:26:02,673 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:03,640 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:04,640 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:05,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:05,789 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:06,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:07,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:08,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:09,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:10,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:11,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:11,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:12,641 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:13,281 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:26:13,281 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 with policy now +2025-01-21 21:26:13,281 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:26:13,282 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:26:13,282 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:26:13,283 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:26:13,545 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/wkmrntx1-media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 +2025-01-21 21:26:13,642 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:14,202 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:26:14,203 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:26:14,203 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 +2025-01-21 21:26:14,204 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval +2025-01-21 21:26:14,204 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media +2025-01-21 21:26:14,204 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos +2025-01-21 21:26:14,512 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/cot1kqu8-media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 +2025-01-21 21:26:14,642 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:15,642 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:16,643 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:16,895 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:17,288 DEBUG SystemMonitor:2650471 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:26:17,289 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:26:17,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:26:17,629 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:26:17,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:18,204 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:26:18,642 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:19,645 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:20,643 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:21,643 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:22,205 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:26:22,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:22,643 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:23,643 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:24,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:25,645 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:26,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:27,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:27,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:28,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:29,208 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:26:29,487 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/tzl4cl22-events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:26:29,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:30,208 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:26:30,645 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:31,650 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:32,209 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:26:32,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:26:32,630 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:26:32,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:32,743 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:33,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:34,209 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:26:34,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:35,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:36,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:37,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:38,632 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:38,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:39,646 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:40,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:41,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:42,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:43,632 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:43,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:44,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:45,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:46,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:47,298 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:26:47,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:26:47,630 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:26:47,673 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:48,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:48,789 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:49,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:50,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:51,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:52,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:53,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:54,633 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:26:54,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:55,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:56,649 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:57,649 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:58,649 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:26:59,649 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:00,633 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:00,649 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:01,649 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:02,629 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:27:02,630 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:27:02,673 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:03,650 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:04,650 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:05,650 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:05,771 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:06,650 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:07,653 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:08,101 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:27:08,102 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 with policy now +2025-01-21 21:27:08,102 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:27:08,102 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:27:08,103 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:27:08,105 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:27:08,220 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:27:08,220 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:08,220 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 +2025-01-21 21:27:08,221 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval +2025-01-21 21:27:08,379 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/vzmwqlwa-media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 +2025-01-21 21:27:08,650 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/e7so9tju-media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 +2025-01-21 21:27:08,658 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:09,663 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:10,221 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:10,659 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:11,128 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:11,660 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:12,661 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:13,660 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:14,223 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:14,662 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:15,661 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:16,223 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:16,224 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:27:16,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:16,921 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:17,058 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:27:17,058 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:27:17,058 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:27:17,224 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:27:17,290 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:27:17,634 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:27:17,634 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:27:17,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:18,224 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:18,667 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:19,693 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:20,663 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:21,663 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:22,226 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:22,226 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:22,663 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:23,663 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:24,663 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:25,663 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:26,664 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:27,637 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:27,664 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:28,664 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:29,664 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:30,664 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:31,664 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:32,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:27:32,630 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:27:32,673 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:32,737 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:33,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:34,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:35,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:36,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:37,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:38,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:38,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:39,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:40,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:41,665 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:42,232 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:42,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:43,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:44,638 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:44,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:45,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:46,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:47,290 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:27:47,630 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:27:47,630 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:27:47,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:48,234 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:48,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:49,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:49,724 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:50,666 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:51,667 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:52,667 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:53,667 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:54,236 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:54,667 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:55,599 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:27:55,667 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:56,607 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:27:56,607 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 with policy now +2025-01-21 21:27:56,609 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:27:56,610 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:27:56,610 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:27:56,614 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:27:56,677 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:56,871 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/6y9lw6pv-media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 +2025-01-21 21:27:57,237 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:27:57,237 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 +2025-01-21 21:27:57,238 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval +2025-01-21 21:27:57,536 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/8b6uehix-media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 +2025-01-21 21:27:57,670 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:58,238 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:27:58,672 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:27:59,673 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:00,673 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:01,297 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:01,678 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:02,239 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:02,635 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:28:02,635 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:28:02,677 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:03,676 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:04,240 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:04,674 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:05,676 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:06,677 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:07,090 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:07,678 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:08,241 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:28:08,503 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/x7ibo417-events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:28:08,675 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:08,898 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:28:08,898 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:28:08,898 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:28:09,241 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:28:09,241 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:28:09,680 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:10,242 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:10,679 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:11,682 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:12,117 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:12,682 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:13,680 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:14,243 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:14,680 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:15,680 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:16,244 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:16,680 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:17,291 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:28:17,291 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:17,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:28:17,631 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:28:17,680 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:18,680 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:19,681 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:20,681 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:21,681 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:22,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:22,681 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:23,681 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:24,681 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:25,681 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:26,682 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:27,682 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:28,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:28,682 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:29,682 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:30,683 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:31,683 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:32,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:28:32,631 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:28:32,683 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:33,683 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:33,754 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:34,683 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:35,685 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:36,684 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:37,684 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:38,690 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:39,644 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:39,688 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:40,688 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:41,689 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:42,733 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:43,692 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:44,253 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:44,691 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:44,734 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:45,691 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:46,691 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:47,291 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:28:47,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:28:47,631 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:28:47,691 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:48,691 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:49,707 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:49,805 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:50,124 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:28:50,125 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 with policy now +2025-01-21 21:28:50,126 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:28:50,128 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:28:50,128 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:28:50,139 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:28:50,255 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:28:50,255 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:50,255 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 +2025-01-21 21:28:50,256 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval +2025-01-21 21:28:50,402 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/pq6izpce-media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 +2025-01-21 21:28:50,706 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:50,706 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/jxcyo4ia-media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 +2025-01-21 21:28:51,705 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:52,256 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:52,704 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:53,706 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:54,708 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:55,707 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:55,805 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:28:56,257 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:56,706 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:57,713 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:58,258 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:28:58,713 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:28:59,714 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:00,714 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:00,906 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:00,959 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:29:00,959 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:29:00,960 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:29:01,259 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:29:01,259 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:29:01,710 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:02,259 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:02,633 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:29:02,634 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:29:02,717 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:03,726 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:04,260 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:04,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:05,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:06,261 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:06,307 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:06,722 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:07,717 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:08,262 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:08,717 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:09,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:10,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:11,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:12,025 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:12,263 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:12,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:13,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:14,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:15,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:16,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:17,291 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:29:17,293 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:17,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:29:17,632 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:29:17,718 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:18,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:19,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:20,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:21,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:22,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:22,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:23,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:24,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:25,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:26,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:27,647 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:27,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:28,719 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:29,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:30,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:31,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:32,631 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:29:32,631 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:29:32,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:32,775 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:33,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:34,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:35,720 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:36,723 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:37,721 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:38,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:38,721 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:39,721 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:40,721 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:41,739 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:42,249 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:29:42,249 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 with policy now +2025-01-21 21:29:42,250 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:29:42,250 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:29:42,251 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:29:42,277 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:29:42,277 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:42,277 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 +2025-01-21 21:29:42,277 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval +2025-01-21 21:29:42,277 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:29:42,536 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/r0tc60x5-media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 +2025-01-21 21:29:42,737 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:42,797 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/x2euuyc5-media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 +2025-01-21 21:29:43,648 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:43,741 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:44,277 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:44,743 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:45,741 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:46,746 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:47,292 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:29:47,642 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:29:47,642 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:29:47,754 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:48,279 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:48,708 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:48,757 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:49,756 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:50,280 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:29:50,754 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:51,759 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:52,759 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:53,758 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:54,659 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:29:54,759 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:55,768 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:56,767 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:57,765 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:58,764 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:29:59,776 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:00,056 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:30:00,056 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:00,056 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:00,057 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:00,283 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:30:00,283 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:30:00,283 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:30:00,527 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:30:00,527 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:30:00,554 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/pvin124k-events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:30:00,686 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:30:00,763 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:01,049 INFO wandb-upload_0:2650471 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpg6369ocs +2025-01-21 21:30:01,070 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:30:01,086 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:30:01,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:30:01,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-21 21:30:01,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 21:30:01,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media +2025-01-21 21:30:01,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table +2025-01-21 21:30:01,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval +2025-01-21 21:30:01,614 INFO SenderThread:2650471 [sender.py:send_request_log_artifact():1525] logged artifact run-0e7f0awm-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3Mjk4Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5NjQ1MQ==', 'latestArtifact': None}} +2025-01-21 21:30:01,614 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:30:01,614 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-21 21:30:01,614 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:30:01,614 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,615 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,615 DEBUG SenderThread:2650471 [sender.py:send():379] send: telemetry +2025-01-21 21:30:01,615 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,616 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,616 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,616 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,616 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,616 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,617 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,617 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,617 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,617 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,618 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,618 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,618 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,618 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,618 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,619 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,619 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,619 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,619 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,620 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,620 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:01,620 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:01,620 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:30:01,763 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:01,803 INFO wandb-upload_1:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/vahap6lh-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 21:30:01,852 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/ciug7oaw-media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-21 21:30:02,040 INFO wandb-upload_0:2650471 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzvvzrdlq +2025-01-21 21:30:02,091 INFO wandb-upload_1:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/hyogeujo-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 21:30:02,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:30:02,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:30:02,462 INFO SenderThread:2650471 [sender.py:send_request_log_artifact():1525] logged artifact run-0e7f0awm-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3Mjk5OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5NjQ1NA==', 'latestArtifact': None}} +2025-01-21 21:30:02,462 DEBUG SenderThread:2650471 [sender.py:send():379] send: files +2025-01-21 21:30:02,462 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json with policy now +2025-01-21 21:30:02,462 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:30:02,462 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:02,463 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:02,632 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:30:02,632 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:30:02,740 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/myjeghn4-media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-21 21:30:02,765 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:03,284 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:30:03,764 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:04,285 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:30:04,764 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:05,698 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:05,764 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:06,764 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:07,764 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:08,765 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:09,765 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:10,765 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:11,661 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:11,780 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:12,288 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/config.yaml +2025-01-21 21:30:12,766 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:13,765 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:14,765 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:15,766 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:16,766 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:17,292 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:30:17,293 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:17,633 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:30:17,633 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:30:17,766 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:18,165 DEBUG SenderThread:2650471 [sender.py:send():379] send: exit +2025-01-21 21:30:18,166 INFO SenderThread:2650471 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:30:18,166 INFO SenderThread:2650471 [sender.py:send_exit():588] handling runtime: 300 +2025-01-21 21:30:18,166 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:18,166 INFO SenderThread:2650471 [sender.py:send_exit():594] send defer +2025-01-21 21:30:18,166 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:18,166 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:30:18,166 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:18,166 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:30:18,167 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:30:18,167 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:18,167 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:30:18,167 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:18,167 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:30:18,167 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:30:18,167 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:18,167 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:30:18,167 INFO HandlerThread:2650471 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:30:18,167 INFO HandlerThread:2650471 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:30:18,168 DEBUG SystemMonitor:2650471 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:30:18,168 INFO HandlerThread:2650471 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:30:18,168 DEBUG SystemMonitor:2650471 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:30:18,222 INFO HandlerThread:2650471 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:30:18,222 INFO HandlerThread:2650471 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:30:18,222 INFO HandlerThread:2650471 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:30:18,223 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:18,223 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:30:18,223 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:30:18,223 DEBUG SenderThread:2650471 [sender.py:send():379] send: stats +2025-01-21 21:30:18,223 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:18,223 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:30:18,223 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:18,223 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:30:18,223 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:30:18,223 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:18,223 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:30:18,290 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:30:20,291 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:30:25,063 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:30:25,063 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:25,063 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:25,064 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:30:25,064 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:30:25,064 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:30:25,065 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:30:25,065 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:25,065 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:30:25,066 DEBUG SenderThread:2650471 [sender.py:send():379] send: history +2025-01-21 21:30:25,066 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:30:25,066 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:25,066 DEBUG SenderThread:2650471 [sender.py:send():379] send: summary +2025-01-21 21:30:25,067 INFO SenderThread:2650471 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:30:25,067 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:25,067 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:30:25,067 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:30:25,067 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:25,067 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:30:25,067 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:25,067 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:30:25,068 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:30:25,068 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:25,068 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:25,068 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:30:25,068 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:25,068 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:30:25,167 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:30:25,293 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:30:26,085 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:30:26,085 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:30:26,085 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:26,085 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:30:26,086 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:26,086 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:30:26,086 INFO SenderThread:2650471 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:30:26,088 INFO SenderThread:2650471 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:30:26,093 INFO SenderThread:2650471 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:30:26,107 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:30:26,108 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:26,108 DEBUG SenderThread:2650471 [sender.py:send():379] send: artifact +2025-01-21 21:30:26,108 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:30:26,167 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:30:26,294 INFO Thread-12 :2650471 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:30:26,513 INFO wandb-upload_0:2650471 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgla44l70 +2025-01-21 21:30:26,609 INFO wandb-upload_1:2650471 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzfmbo_92 +2025-01-21 21:30:26,660 INFO wandb-upload_2:2650471 [upload_job.py:push():88] Uploaded file /tmp/tmp8vwc9knz/wandb-job.json +2025-01-21 21:30:27,273 INFO SenderThread:2650471 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3MzQyMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'versionIndex': 454}}} +2025-01-21 21:30:27,273 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:27,273 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:30:27,273 INFO SenderThread:2650471 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:30:27,295 INFO SenderThread:2650471 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files +2025-01-21 21:30:27,295 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/config.yaml config.yaml +2025-01-21 21:30:27,295 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json wandb-summary.json +2025-01-21 21:30:27,295 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/requirements.txt requirements.txt +2025-01-21 21:30:27,295 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:30:27,295 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:30:27,295 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log output.log +2025-01-21 21:30:27,296 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/diff.patch diff.patch +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 media/videos/eval/4_eval_4_1dd664735f9b5c78730c.mp4 +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 media/videos/eval/3_eval_3_6c7d164a44f31be2687d.mp4 +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 media/videos/eval/1_eval_1_6b41bcf0751cacefe8bb.mp4 +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 media/videos/eval/0_eval_0_43ff0adeaa2aac56e705.mp4 +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 media/videos/eval/2_eval_2_ebcc91651af67cca62de.mp4 +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json media/table/eval/total_success_rate_ci_6_e3405f8336f17a4a59dd.table.json +2025-01-21 21:30:27,301 INFO SenderThread:2650471 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 21:30:27,302 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:30:27,302 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:30:27,302 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:27,302 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:30:27,302 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:27,304 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:30:27,305 INFO SenderThread:2650471 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:30:27,480 INFO wandb-upload_1:2650471 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/wandb-summary.json +2025-01-21 21:30:27,486 INFO wandb-upload_2:2650471 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/requirements.txt +2025-01-21 21:30:27,507 INFO wandb-upload_0:2650471 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/config.yaml +2025-01-21 21:30:27,586 INFO wandb-upload_4:2650471 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/files/output.log +2025-01-21 21:30:27,641 INFO wandb-upload_3:2650471 [upload_job.py:push():130] Uploaded file /tmp/tmp6mx1mwgwwandb/rm9m3ufd-events.out.tfevents.1737512717.leonmkim-ROG-Strix-G15CS-G15CS.2650272.0 +2025-01-21 21:30:27,842 INFO Thread-11 (_thread_body):2650471 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:30:27,842 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:27,842 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:30:27,842 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:27,842 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:30:27,842 INFO SenderThread:2650471 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:30:27,844 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:30:27,844 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:27,844 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:30:27,844 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:27,844 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:30:27,844 INFO SenderThread:2650471 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:30:28,069 INFO SenderThread:2650471 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:30:28,069 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:30:28,069 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:28,069 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:30:28,069 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:28,069 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:30:28,069 INFO SenderThread:2650471 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:30:28,069 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:30:28,069 INFO HandlerThread:2650471 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:30:28,070 DEBUG SenderThread:2650471 [sender.py:send():379] send: final +2025-01-21 21:30:28,070 DEBUG SenderThread:2650471 [sender.py:send():379] send: footer +2025-01-21 21:30:28,070 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: defer +2025-01-21 21:30:28,070 INFO SenderThread:2650471 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:30:28,070 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:30:28,070 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:30:28,071 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:30:28,071 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:30:28,072 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:30:28,072 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:30:28,073 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:30:28,073 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:28,126 DEBUG SenderThread:2650471 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:30:28,126 INFO MainThread:2650471 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:30:28,126 INFO MainThread:2650471 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:30:28,127 INFO MainThread:2650471 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:30:28,127 DEBUG HandlerThread:2650471 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:30:28,127 INFO HandlerThread:2650471 [handler.py:finish():882] shutting down handler +2025-01-21 21:30:29,072 INFO WriterThread:2650471 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/run-0e7f0awm.wandb +2025-01-21 21:30:29,126 INFO SenderThread:2650471 [sender.py:finish():1615] shutting down sender +2025-01-21 21:30:29,126 INFO SenderThread:2650471 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:30:29,126 INFO SenderThread:2650471 [file_pusher.py:join():175] waiting for file pusher diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/logs/debug.log b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8b7f073f1bafbce91679ebeb90493507fbe48233 --- /dev/null +++ b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Configure stats pid to 2650272 +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/settings +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:25:16,834 INFO MainThread:2650272 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/logs/debug.log +2025-01-21 21:25:16,835 INFO MainThread:2650272 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/logs/debug-internal.log +2025-01-21 21:25:16,835 INFO MainThread:2650272 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:25:16,835 INFO MainThread:2650272 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_1', 'wandb_notes': '4295_1_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm'} +2025-01-21 21:25:16,835 INFO MainThread:2650272 [wandb_init.py:init():619] starting backend +2025-01-21 21:25:16,835 INFO MainThread:2650272 [wandb_init.py:init():623] setting up manager +2025-01-21 21:25:16,839 INFO MainThread:2650272 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:25:16,840 INFO MainThread:2650272 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:25:16,854 INFO MainThread:2650272 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:25:16,860 INFO MainThread:2650272 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:25:17,200 INFO MainThread:2650272 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:25:17,274 INFO MainThread:2650272 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:25:17,275 INFO MainThread:2650272 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:25:17,629 INFO MainThread:2650272 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:25:17,629 INFO MainThread:2650272 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:25:17,629 INFO MainThread:2650272 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:25:17,629 INFO MainThread:2650272 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:25:17,631 INFO MainThread:2650272 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:25:17,631 INFO MainThread:2650272 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/0e7f0awm/tb, True +2025-01-21 21:25:24,844 INFO MainThread:2650272 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:30:29,128 WARNING MsgRouterThr:2650272 [router.py:message_loop():77] message_loop has been closed diff --git a/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/run-0e7f0awm.wandb b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/run-0e7f0awm.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d9a2b2d1b0a31fddbd58079d3f21a999cd45194e Binary files /dev/null and b/0e7f0awm/wandb/run-20250121_212516-0e7f0awm/run-0e7f0awm.wandb differ diff --git a/4lg2e533/.hydra/config.yaml b/4lg2e533/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2154a6acb64e2809a71d0adfc508bd71e0da7dde --- /dev/null +++ b/4lg2e533/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/4lg2e533/.hydra/hydra.yaml b/4lg2e533/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d1155f82d69e46328afdb29a6270b16922a2663 --- /dev/null +++ b/4lg2e533/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/4lg2e533/.hydra/overrides.yaml b/4lg2e533/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/4lg2e533/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/4lg2e533/episode_rosbags/aligned_depth_to_color_K.npy b/4lg2e533/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/4lg2e533/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/4lg2e533/episode_rosbags/cam_tf_world.npy b/4lg2e533/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/4lg2e533/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/4lg2e533/episode_rosbags/color_K.npy b/4lg2e533/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/4lg2e533/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/4lg2e533/episode_rosbags/depth_K.npy b/4lg2e533/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/4lg2e533/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/4lg2e533/eval_robot.log b/4lg2e533/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..285127a26a8b08306da2abdba29d1f946d890966 --- /dev/null +++ b/4lg2e533/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 22:01:03,979][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 22:01:03,982][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/4lg2e533/eval_video/0_eval.mp4 b/4lg2e533/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8059a4a9c3efa34804cdb1a2e2e8901e76156a68 --- /dev/null +++ b/4lg2e533/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0d20ac03e2b6a636fd4b75dc420d38bca78737161b03b456f4bafb0c9d3ac5c +size 1482819 diff --git a/4lg2e533/eval_video/1_eval.mp4 b/4lg2e533/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4f4c6d9dafc8bcedabf4db53a91f329ca0e2eb77 --- /dev/null +++ b/4lg2e533/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c351cf40eb1c20bd3656095d6296b33b5c1d6813b7dc6d200beb36a1247d8eb4 +size 1501132 diff --git a/4lg2e533/eval_video/2_eval.mp4 b/4lg2e533/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9175b9a9a78f38d336e14ac7ce78b7833533ded --- /dev/null +++ b/4lg2e533/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:977e432fb56e9f58fb599098d19508aadfcf98861d2346a062f799302713a2ce +size 1562026 diff --git a/4lg2e533/eval_video/3_eval.mp4 b/4lg2e533/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0e57716ef9c1f27465207e38b3016c192452644 --- /dev/null +++ b/4lg2e533/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1e75bdfd0a4e32aff60e01308eeaf7bec80ee9a3c08c90f23113a3ee4e9300a +size 1508395 diff --git a/4lg2e533/eval_video/4_eval.mp4 b/4lg2e533/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..34855b7f83361dbfb68f336b20c92af56e8f71ed --- /dev/null +++ b/4lg2e533/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac7a1d089ff431aae59a91f891770d23cf42ae6990d7ccf3519ffdfae0139d56 +size 1615658 diff --git a/4lg2e533/tb/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 b/4lg2e533/tb/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 new file mode 100644 index 0000000000000000000000000000000000000000..1b9a51d0610eadd737410a831b97c02ee98fa96c --- /dev/null +++ b/4lg2e533/tb/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3debb0c0f953c8dedc6aaf8a035e26ea6d9811f695061522fba3a9c701bf8bff +size 6539 diff --git a/4lg2e533/wandb/debug-internal.log b/4lg2e533/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f41bff36ef289abf94e47b78b5853b4058df7706 --- /dev/null +++ b/4lg2e533/wandb/debug-internal.log @@ -0,0 +1,858 @@ +2025-01-21 22:01:10,215 INFO StreamThr :2660143 [internal.py:wandb_internal():85] W&B internal server running at pid: 2660143, started at: 2025-01-21 22:01:10.215539 +2025-01-21 22:01:10,216 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status +2025-01-21 22:01:10,232 INFO WriterThread:2660143 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/run-4lg2e533.wandb +2025-01-21 22:01:10,233 DEBUG SenderThread:2660143 [sender.py:send():379] send: header +2025-01-21 22:01:10,240 DEBUG SenderThread:2660143 [sender.py:send():379] send: run +2025-01-21 22:01:10,481 INFO SenderThread:2660143 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files +2025-01-21 22:01:10,481 INFO SenderThread:2660143 [sender.py:_start_run_threads():1188] run started: 4lg2e533 with start time 1737514870.21678 +2025-01-21 22:01:10,496 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 22:01:10,496 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: check_version +2025-01-21 22:01:10,570 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 22:01:10,575 DEBUG HandlerThread:2660143 [system_info.py:__init__():26] System info init +2025-01-21 22:01:10,575 DEBUG HandlerThread:2660143 [system_info.py:__init__():41] System info init done +2025-01-21 22:01:10,575 INFO HandlerThread:2660143 [system_monitor.py:start():194] Starting system monitor +2025-01-21 22:01:10,575 INFO SystemMonitor:2660143 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 22:01:10,576 INFO HandlerThread:2660143 [system_monitor.py:probe():214] Collecting system info +2025-01-21 22:01:10,576 INFO SystemMonitor:2660143 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 22:01:10,577 INFO SystemMonitor:2660143 [interfaces.py:start():188] Started disk monitoring +2025-01-21 22:01:10,577 INFO SystemMonitor:2660143 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 22:01:10,579 INFO SystemMonitor:2660143 [interfaces.py:start():188] Started memory monitoring +2025-01-21 22:01:10,579 INFO SystemMonitor:2660143 [interfaces.py:start():188] Started network monitoring +2025-01-21 22:01:10,589 DEBUG HandlerThread:2660143 [system_info.py:probe():152] Probing system +2025-01-21 22:01:10,591 DEBUG HandlerThread:2660143 [system_info.py:_probe_git():137] Probing git +2025-01-21 22:01:10,596 DEBUG HandlerThread:2660143 [system_info.py:_probe_git():145] Probing git done +2025-01-21 22:01:10,596 DEBUG HandlerThread:2660143 [system_info.py:probe():200] Probing system done +2025-01-21 22:01:10,596 DEBUG HandlerThread:2660143 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T03:01:10.590028', 'startedAt': '2025-01-22T03:01:10.206489', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3747.7392499999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.155, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 437.4647331237793}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 22:01:10,596 INFO HandlerThread:2660143 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 22:01:10,596 INFO HandlerThread:2660143 [system_monitor.py:probe():227] Publishing system info +2025-01-21 22:01:10,596 DEBUG HandlerThread:2660143 [system_info.py:_save_code():44] Saving code +2025-01-21 22:01:10,597 DEBUG HandlerThread:2660143 [system_info.py:_save_code():67] Saving code done +2025-01-21 22:01:10,597 DEBUG HandlerThread:2660143 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 22:01:10,659 DEBUG HandlerThread:2660143 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 22:01:10,660 INFO HandlerThread:2660143 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 22:01:10,663 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:01:10,663 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 22:01:10,663 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 22:01:10,664 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 22:01:10,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 22:01:10,919 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/vrmadxke-wandb-metadata.json +2025-01-21 22:01:10,919 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: python_packages +2025-01-21 22:01:10,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:01:10,920 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:01:10,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:10,924 INFO HandlerThread:2660143 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/tb" + save: true +} +control { + relay_id: "fbf885f55bfb41ae9fd24d9eb85e2419" +} +_info { + stream_id: "4lg2e533" +} + +2025-01-21 22:01:10,924 DEBUG HandlerThread:2660143 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 22:01:10,938 INFO wandb-upload_2:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/fpn7u3em-diff.patch +2025-01-21 22:01:10,950 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/2hautu9g-code/FISH/eval_robot.py +2025-01-21 22:01:11,009 DEBUG SenderThread:2660143 [sender.py:send():379] send: telemetry +2025-01-21 22:01:11,009 DEBUG SenderThread:2660143 [sender.py:send():379] send: tbrecord +2025-01-21 22:01:11,010 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:01:11,010 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 with policy live +2025-01-21 22:01:11,202 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/1n40o3p2-events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:01:11,482 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/diff.patch +2025-01-21 22:01:11,483 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/requirements.txt +2025-01-21 22:01:11,483 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:01:11,483 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-metadata.json +2025-01-21 22:01:11,483 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:11,483 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/code/FISH/eval_robot.py +2025-01-21 22:01:11,483 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/code +2025-01-21 22:01:11,483 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/code/FISH +2025-01-21 22:01:11,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:12,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:13,482 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:13,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:14,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:15,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:15,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:16,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:17,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:18,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:19,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:20,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:20,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:20,975 DEBUG SenderThread:2660143 [sender.py:send():379] send: config +2025-01-21 22:01:21,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:22,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:23,484 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:23,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:24,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:25,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:01:25,919 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:01:25,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:26,070 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:26,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:27,486 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:27,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:28,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:29,486 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:29,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:30,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:31,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:31,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:32,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:33,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:34,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:35,488 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:35,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:36,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:37,489 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:37,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:37,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:38,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:39,489 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:39,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:40,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:01:40,919 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:01:40,961 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:41,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:42,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:43,099 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:43,490 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/config.yaml +2025-01-21 22:01:43,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:44,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:45,491 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:45,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:46,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:47,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:48,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:48,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:49,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:50,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:51,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:52,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:53,493 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:53,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:54,889 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:54,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:55,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:01:55,919 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:01:55,961 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:56,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:57,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:58,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:59,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:59,982 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:00,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:01,495 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:01,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:02,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:03,496 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:03,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:04,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:05,425 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:05,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:06,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:07,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:08,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:09,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:10,579 DEBUG SystemMonitor:2660143 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 22:02:10,580 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:02:10,580 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:10,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:02:10,919 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:02:10,961 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:11,928 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:12,928 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:13,922 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:02:13,922 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:02:13,922 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 with policy now +2025-01-21 22:02:13,923 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:02:13,923 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:02:13,924 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:02:13,929 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:14,171 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/qoja1q6g-media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 +2025-01-21 22:02:14,499 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:02:14,499 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 +2025-01-21 22:02:14,499 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos +2025-01-21 22:02:14,499 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media +2025-01-21 22:02:14,499 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval +2025-01-21 22:02:14,791 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/hwuybd8d-media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 +2025-01-21 22:02:14,928 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:15,499 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:15,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:15,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:16,928 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:17,929 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:18,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:19,501 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:19,929 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:20,929 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:21,070 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:21,501 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:21,501 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:02:21,929 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:22,165 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/cyezw7ny-events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:02:22,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:23,502 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:23,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:24,931 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:25,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:02:25,920 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:02:25,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:26,085 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:26,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:27,503 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:27,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:28,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:29,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:30,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:31,916 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:31,961 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:32,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:33,505 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:33,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:34,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:35,931 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:36,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:36,932 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:37,932 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:38,933 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:39,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:40,580 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:02:40,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:02:40,920 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:02:40,965 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:41,933 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:42,062 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:42,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:43,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:44,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:45,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:46,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:47,922 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:47,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:48,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:49,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:50,935 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:51,936 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:52,922 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:52,935 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:53,936 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:54,936 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:55,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:02:55,920 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:02:55,961 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:56,936 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:57,936 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:58,074 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:58,936 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:59,945 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:00,760 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:03:00,760 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 with policy now +2025-01-21 22:03:00,760 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:03:00,760 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:03:00,761 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:03:00,778 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:03:00,937 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:00,999 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/4wkmge9q-media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 +2025-01-21 22:03:01,515 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:03:01,515 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:01,515 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 +2025-01-21 22:03:01,515 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval +2025-01-21 22:03:01,816 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/m3054ytg-media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 +2025-01-21 22:03:01,938 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:02,940 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:03,516 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:03,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:03,945 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:04,943 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:05,517 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:05,942 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:06,945 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:07,948 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:08,948 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:09,225 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:09,518 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:09,956 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:10,581 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:03:10,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:03:10,923 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:03:10,969 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:11,949 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:12,949 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:13,953 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:14,951 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:15,068 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:15,953 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:16,954 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:17,954 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:18,953 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:19,356 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:03:19,356 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:03:19,357 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:03:19,521 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:03:19,521 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:03:19,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:20,079 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:20,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:21,522 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:21,960 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:22,959 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:23,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:24,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:25,190 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:25,523 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:25,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:03:25,921 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:03:25,954 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:26,954 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:27,525 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:27,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:28,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:29,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:30,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:30,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:31,526 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:31,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:32,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:33,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:34,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:35,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:35,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:36,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:37,956 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:38,956 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:39,956 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:40,581 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:03:40,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:03:40,921 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:03:40,961 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:41,070 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:41,956 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:42,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:43,956 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:44,956 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:45,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:46,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:46,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:47,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:48,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:49,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:50,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:51,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:51,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:52,958 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:53,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:54,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:55,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:03:55,921 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:03:55,965 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:56,958 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:57,056 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:57,958 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:58,958 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:59,962 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:00,410 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:04:00,410 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 with policy now +2025-01-21 22:04:00,413 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:04:00,414 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:04:00,414 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:04:00,427 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:04:00,537 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:04:00,537 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 +2025-01-21 22:04:00,538 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval +2025-01-21 22:04:00,658 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/y0ky4200-media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 +2025-01-21 22:04:00,967 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:00,976 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/vnq1vyz5-media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 +2025-01-21 22:04:01,538 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:01,962 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:02,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:02,969 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:03,965 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:04,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:05,539 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:05,968 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:06,965 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:07,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:08,825 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:08,970 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:09,541 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:09,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:10,582 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:04:10,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:04:10,926 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:04:10,976 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:11,969 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:12,200 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:04:12,200 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:04:12,202 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:04:12,542 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:04:12,542 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:04:12,809 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/q9f1eby7-events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:04:12,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:13,542 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:13,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:14,693 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:14,978 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:15,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:16,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:17,544 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:17,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:18,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:19,544 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:19,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:20,329 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:20,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:21,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:22,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:23,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:24,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:25,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:04:25,922 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:04:25,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:26,010 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:26,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:27,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:28,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:29,973 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:30,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:31,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:31,975 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:32,973 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:33,973 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:34,974 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:35,974 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:36,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:36,974 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:37,974 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:38,978 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:39,975 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:40,582 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:04:40,922 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:04:40,922 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:04:40,975 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:41,975 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:41,985 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:42,976 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:43,975 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:44,975 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:45,975 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:46,976 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:47,928 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:47,976 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:48,976 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:49,976 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:50,976 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:51,979 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:52,143 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:04:52,143 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 with policy now +2025-01-21 22:04:52,155 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:04:52,156 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:04:52,156 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:04:52,388 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:04:52,388 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/zdqszyn5-media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 +2025-01-21 22:04:52,556 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:04:52,556 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 +2025-01-21 22:04:52,556 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval +2025-01-21 22:04:52,851 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/1i68o5ah-media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 +2025-01-21 22:04:52,992 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:53,407 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:53,556 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:53,989 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:54,986 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:55,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:04:55,925 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:04:55,988 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:56,985 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:57,557 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:57,988 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:58,651 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:58,983 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:59,558 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:59,989 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:00,262 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:05:00,262 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:05:00,263 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:05:00,558 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:05:00,558 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:05:01,003 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:01,559 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:01,991 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:02,996 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:03,559 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:03,903 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:03,993 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:04,996 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:05,560 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:05,990 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:06,991 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:07,561 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:07,990 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:08,991 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:09,038 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:09,561 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:09,991 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:10,583 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:05:10,922 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:05:10,923 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:05:10,991 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:11,991 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:12,992 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:13,563 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:13,997 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:14,629 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:14,996 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:15,994 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:16,997 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:17,994 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:18,994 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:19,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:19,995 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:20,995 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:21,994 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:22,995 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:23,995 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:24,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:24,996 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:25,922 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:05:25,922 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:05:26,002 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:27,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:27,999 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:29,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:30,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:30,076 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:31,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:32,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:33,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:34,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:35,001 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:35,935 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:36,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:37,001 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:38,002 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:39,002 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:40,001 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:40,583 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:05:40,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:05:40,924 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:05:41,001 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:41,039 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:42,012 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:42,787 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:05:42,787 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 with policy now +2025-01-21 22:05:42,793 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:05:42,794 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:05:42,794 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:05:42,800 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:05:43,002 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:43,094 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/3hf7alg0-media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 +2025-01-21 22:05:43,573 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:05:43,573 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:43,573 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 +2025-01-21 22:05:43,573 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval +2025-01-21 22:05:43,888 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/8ee5ptx9-media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 +2025-01-21 22:05:44,015 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:45,007 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:45,574 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:46,015 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:46,441 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:47,012 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:47,574 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:48,014 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:49,014 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:50,014 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:51,013 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:51,576 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:51,945 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:52,017 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:53,016 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:54,013 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:55,013 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:55,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:05:55,934 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:05:56,023 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:57,021 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:57,073 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:58,016 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:59,018 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:00,017 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:01,026 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:02,022 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:02,953 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:03,030 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:04,021 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:05,020 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:06,029 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:06,340 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:06:06,341 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:06,342 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:06,581 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:06:06,581 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:06:06,832 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/j3cwuhh8-events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:06:07,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:07,283 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:06:07,284 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:06:07,445 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:06:07,581 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:06:07,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:06:07,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media +2025-01-21 22:06:07,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table +2025-01-21 22:06:07,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval +2025-01-21 22:06:07,762 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/yducq79t-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:06:07,845 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:06:07,855 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:06:07,880 INFO wandb-upload_1:2660143 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp7ay50xjm +2025-01-21 22:06:08,024 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:08,359 INFO SenderThread:2660143 [sender.py:send_request_log_artifact():1525] logged artifact run-4lg2e533-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcxNTI2NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwMzU1OA==', 'latestArtifact': None}} +2025-01-21 22:06:08,359 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:06:08,359 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-21 22:06:08,359 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:06:08,359 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,360 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:08,360 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,361 DEBUG SenderThread:2660143 [sender.py:send():379] send: telemetry +2025-01-21 22:06:08,361 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,365 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,365 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,366 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,366 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,366 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,366 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,367 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,367 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,367 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,367 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,367 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,368 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,368 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,368 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,368 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,368 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,369 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,369 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,369 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,369 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,370 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,370 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:06:08,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:06:08,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:06:08,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-21 22:06:08,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval +2025-01-21 22:06:08,621 INFO wandb-upload_2:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/143fr4um-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:06:08,785 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/wrrc4rhk-media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-21 22:06:09,027 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:09,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:06:09,584 INFO wandb-upload_0:2660143 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgkkabjf_ +2025-01-21 22:06:10,024 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:10,433 INFO SenderThread:2660143 [sender.py:send_request_log_artifact():1525] logged artifact run-4lg2e533-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcxNTI4Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwMzU2MA==', 'latestArtifact': None}} +2025-01-21 22:06:10,433 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:06:10,433 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json with policy now +2025-01-21 22:06:10,434 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:06:10,434 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:10,435 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:10,583 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:06:10,584 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:06:10,678 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/5dlufjw4-media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-21 22:06:10,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:06:10,925 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:06:11,024 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:12,026 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:13,025 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:14,012 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:14,025 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:15,025 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:16,025 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:17,026 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:18,026 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:19,030 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:19,945 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:20,027 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:21,027 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:22,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:23,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:24,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:24,945 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:25,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:25,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:06:25,926 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:06:26,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:27,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:28,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:29,029 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:30,030 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:30,077 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:31,032 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:32,029 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:33,029 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:34,032 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:35,030 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:35,948 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:36,046 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:36,593 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/config.yaml +2025-01-21 22:06:37,031 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:38,031 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:39,031 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:40,031 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:40,584 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:06:40,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:06:40,926 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:06:41,033 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:41,244 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:41,517 DEBUG SenderThread:2660143 [sender.py:send():379] send: exit +2025-01-21 22:06:41,517 INFO SenderThread:2660143 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 22:06:41,518 INFO SenderThread:2660143 [sender.py:send_exit():588] handling runtime: 330 +2025-01-21 22:06:41,518 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:41,518 INFO SenderThread:2660143 [sender.py:send_exit():594] send defer +2025-01-21 22:06:41,518 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:41,518 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 22:06:41,518 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:41,518 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 22:06:41,518 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 1 +2025-01-21 22:06:41,518 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:41,519 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 22:06:41,519 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:41,519 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 22:06:41,519 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 2 +2025-01-21 22:06:41,519 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:41,519 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 22:06:41,519 INFO HandlerThread:2660143 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 22:06:41,519 DEBUG SystemMonitor:2660143 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 22:06:41,519 DEBUG SystemMonitor:2660143 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 22:06:41,519 INFO HandlerThread:2660143 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 22:06:41,520 INFO HandlerThread:2660143 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 22:06:41,571 INFO HandlerThread:2660143 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 22:06:41,571 INFO HandlerThread:2660143 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 22:06:41,571 INFO HandlerThread:2660143 [interfaces.py:finish():200] Joined network monitor +2025-01-21 22:06:41,572 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:41,572 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 22:06:41,572 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 3 +2025-01-21 22:06:41,572 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:06:41,572 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:41,572 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 22:06:41,573 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:41,573 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 22:06:41,573 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 4 +2025-01-21 22:06:41,573 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:41,573 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 22:06:41,595 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:06:43,596 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:06:48,352 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:06:48,353 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:48,353 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:48,353 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 22:06:48,353 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 5 +2025-01-21 22:06:48,353 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 22:06:48,353 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:06:48,354 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:48,354 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:06:48,354 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 22:06:48,355 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:48,355 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:48,356 DEBUG SenderThread:2660143 [sender.py:send():379] send: summary +2025-01-21 22:06:48,360 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:48,360 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:48,360 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 22:06:48,360 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 6 +2025-01-21 22:06:48,360 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:48,360 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 22:06:48,360 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:48,360 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 22:06:48,360 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 7 +2025-01-21 22:06:48,360 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:48,361 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:48,361 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 22:06:48,361 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:48,361 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 22:06:48,519 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:06:48,598 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:06:50,837 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 8 +2025-01-21 22:06:50,837 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:06:50,837 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:50,838 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 22:06:50,838 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:50,838 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 22:06:50,838 INFO SenderThread:2660143 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 22:06:50,841 INFO SenderThread:2660143 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 22:06:50,843 INFO SenderThread:2660143 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 22:06:50,856 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 9 +2025-01-21 22:06:50,857 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:50,857 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 22:06:50,857 DEBUG SenderThread:2660143 [sender.py:send():379] send: artifact +2025-01-21 22:06:51,136 INFO SenderThread:2660143 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3MzQyMw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 22:06:51,136 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:51,136 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 22:06:51,136 INFO SenderThread:2660143 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 22:06:51,520 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:06:51,599 INFO SenderThread:2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:06:51,599 INFO SenderThread:2660143 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files +2025-01-21 22:06:51,599 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/config.yaml config.yaml +2025-01-21 22:06:51,599 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json wandb-summary.json +2025-01-21 22:06:51,599 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/requirements.txt requirements.txt +2025-01-21 22:06:51,599 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-metadata.json wandb-metadata.json +2025-01-21 22:06:51,600 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:06:51,603 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log output.log +2025-01-21 22:06:51,605 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/diff.patch diff.patch +2025-01-21 22:06:51,607 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 22:06:51,608 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 +2025-01-21 22:06:51,608 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 +2025-01-21 22:06:51,608 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 +2025-01-21 22:06:51,609 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 +2025-01-21 22:06:51,609 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 +2025-01-21 22:06:51,609 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-21 22:06:51,609 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:06:51,609 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 10 +2025-01-21 22:06:51,609 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:06:51,609 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:51,609 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 22:06:51,610 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:51,610 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 22:06:51,610 INFO SenderThread:2660143 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:06:51,780 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/config.yaml +2025-01-21 22:06:51,818 INFO wandb-upload_2:2660143 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/requirements.txt +2025-01-21 22:06:51,839 INFO wandb-upload_4:2660143 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:06:51,839 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:06:51,910 INFO wandb-upload_3:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/fosgc2nr-events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:06:52,110 INFO Thread-11 (_thread_body):2660143 [sender.py:transition_state():614] send defer: 11 +2025-01-21 22:06:52,110 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:52,110 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 22:06:52,110 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:52,111 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 22:06:52,111 INFO SenderThread:2660143 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 22:06:52,112 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 12 +2025-01-21 22:06:52,112 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:52,112 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 22:06:52,112 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:52,112 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 22:06:52,112 INFO SenderThread:2660143 [file_stream.py:finish():601] file stream finish called +2025-01-21 22:06:52,322 INFO SenderThread:2660143 [file_stream.py:finish():605] file stream finish is done +2025-01-21 22:06:52,322 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 13 +2025-01-21 22:06:52,323 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:52,323 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 22:06:52,323 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:52,323 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 22:06:52,323 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 14 +2025-01-21 22:06:52,323 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:52,323 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 22:06:52,324 DEBUG SenderThread:2660143 [sender.py:send():379] send: final +2025-01-21 22:06:52,324 DEBUG SenderThread:2660143 [sender.py:send():379] send: footer +2025-01-21 22:06:52,324 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:52,324 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 22:06:52,324 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:06:52,324 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:06:52,325 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 22:06:52,325 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: server_info +2025-01-21 22:06:52,326 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:06:52,327 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 22:06:52,327 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 22:06:52,328 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:52,386 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:06:52,386 INFO MainThread:2660143 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 22:06:52,387 INFO MainThread:2660143 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 22:06:52,387 INFO MainThread:2660143 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 22:06:52,387 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 22:06:52,387 INFO HandlerThread:2660143 [handler.py:finish():882] shutting down handler +2025-01-21 22:06:53,327 INFO WriterThread:2660143 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/run-4lg2e533.wandb +2025-01-21 22:06:53,386 INFO SenderThread:2660143 [sender.py:finish():1615] shutting down sender +2025-01-21 22:06:53,387 INFO SenderThread:2660143 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:06:53,387 INFO SenderThread:2660143 [file_pusher.py:join():175] waiting for file pusher diff --git a/4lg2e533/wandb/debug.log b/4lg2e533/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d5ff3cd5d214c7be4fb3fac7136c8156bf0885f8 --- /dev/null +++ b/4lg2e533/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Configure stats pid to 2659946 +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/settings +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/logs/debug.log +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/logs/debug-internal.log +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_init.py:init():569] calling init triggers +2025-01-21 22:01:10,209 INFO MainThread:2659946 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_1', 'wandb_notes': '4295_1_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533'} +2025-01-21 22:01:10,209 INFO MainThread:2659946 [wandb_init.py:init():619] starting backend +2025-01-21 22:01:10,209 INFO MainThread:2659946 [wandb_init.py:init():623] setting up manager +2025-01-21 22:01:10,214 INFO MainThread:2659946 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 22:01:10,216 INFO MainThread:2659946 [wandb_init.py:init():631] backend started and connected +2025-01-21 22:01:10,232 INFO MainThread:2659946 [wandb_init.py:init():720] updated telemetry +2025-01-21 22:01:10,238 INFO MainThread:2659946 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 22:01:10,495 INFO MainThread:2659946 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 22:01:10,566 INFO MainThread:2659946 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 22:01:10,567 INFO MainThread:2659946 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 22:01:10,919 INFO MainThread:2659946 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 22:01:10,920 INFO MainThread:2659946 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 22:01:10,921 INFO MainThread:2659946 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 22:01:10,921 INFO MainThread:2659946 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 22:01:10,923 INFO MainThread:2659946 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 22:01:10,924 INFO MainThread:2659946 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/tb, True +2025-01-21 22:01:20,975 INFO MainThread:2659946 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 22:06:53,388 WARNING MsgRouterThr:2659946 [router.py:message_loop():77] message_loop has been closed diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/code/FISH/eval_robot.py b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/config.yaml b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69cba7751b04d284d13c30ba2340dcd8e5f6b95c --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_1' +wandb_notes: + desc: null + value: 4295_1_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737514870 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/diff.patch b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..0d1d1c2d34392be0466354e2f5a3da5c22dfe3f1 --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/diff.patch @@ -0,0 +1,24 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..3a697a3 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False +@@ -144,8 +144,8 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '3465_2' # seed 2 + + # crp_D+mask+contact+act history +-wandb_run_id: '4295_0' +-# wandb_run_id: '4295_1' # seed 1 ++# wandb_run_id: '4295_0' ++wandb_run_id: '4295_1' # seed 1 + # wandb_run_id: '4295_2' # seed 2 + + # RGBD+mask+act history diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json new file mode 100644 index 0000000000000000000000000000000000000000..5271efacb3db6a0813497da4611af43307900d95 --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[15, 15, 1.0, 0.9928352235762072, 1, 0.7993280141947043, 1, 0.8249338715618673, 1, 1, 1]]} \ No newline at end of file diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8059a4a9c3efa34804cdb1a2e2e8901e76156a68 --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0d20ac03e2b6a636fd4b75dc420d38bca78737161b03b456f4bafb0c9d3ac5c +size 1482819 diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4f4c6d9dafc8bcedabf4db53a91f329ca0e2eb77 --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c351cf40eb1c20bd3656095d6296b33b5c1d6813b7dc6d200beb36a1247d8eb4 +size 1501132 diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9175b9a9a78f38d336e14ac7ce78b7833533ded --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:977e432fb56e9f58fb599098d19508aadfcf98861d2346a062f799302713a2ce +size 1562026 diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0e57716ef9c1f27465207e38b3016c192452644 --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1e75bdfd0a4e32aff60e01308eeaf7bec80ee9a3c08c90f23113a3ee4e9300a +size 1508395 diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..34855b7f83361dbfb68f336b20c92af56e8f71ed --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac7a1d089ff431aae59a91f891770d23cf42ae6990d7ccf3519ffdfae0139d56 +size 1615658 diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..33955822effeef0ae1b7d760f03257d749bb1722 --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log @@ -0,0 +1,272 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737514880.975662]: resetting environment +[INFO] [1737514880.979589]: cleared current plan +[INFO] [1737514880.980525]: moving to home +[INFO] [1737514881.982567]: reached home +[INFO] [1737514881.984111]: reset action history +[INFO] [1737514884.695597]: environment reset +Starting episode 0 +[INFO] [1737514884.700174]: resetting environment +[INFO] [1737514884.704246]: cleared current plan +[INFO] [1737514884.705206]: moving to home +[INFO] [1737514885.707458]: reached home +[INFO] [1737514885.708777]: reset action history +[INFO] [1737514888.418306]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737514894.431144]: resetting environment +[INFO] [1737514894.435470]: cleared current plan +[INFO] [1737514894.436367]: moving to home +[INFO] [1737514895.438464]: reached home +[INFO] [1737514895.439718]: reset action history +[INFO] [1737514898.147976]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/episode_rosbags/episode_0_2025-01-21-22-01-38.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2659946.log +started roslaunch server http://158.130.50.37:43851/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2660252] +process[print_text-2]: started with pid [2660276] +process[rosbag_record-3]: started with pid [2660277] +started bagging! +For topic gripper_width: timestamp difference is -113174921 for nearest: 1737514903912473911 - target: 1737514904025648832 at idx 88 +For topic gripper_width: timestamp difference is -150456972 for nearest: 1737514911612469368 - target: 1737514911762926340 at idx 179 +For topic gripper_width: timestamp difference is 100153536 for nearest: 1737514920312496990 - target: 1737514920212343454 at idx 249 +For topic gripper_width: timestamp difference is 104382075 for nearest: 1737514922379160680 - target: 1737514922274778605 at idx 249 +For topic gripper_width: timestamp difference is 107916469 for nearest: 1737514932679139013 - target: 1737514932571222544 at idx 249 +[INFO] [1737514932.919501]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737514933.691333]: Stored episode 1. +[INFO] [1737514933.691630]: Saving video... +[INFO] [1737514933.911437]: Video saved! +Starting episode 1 +[INFO] [1737514933.922851]: resetting environment +[INFO] [1737514933.938324]: cleared current plan +[INFO] [1737514933.938611]: moving to home +[INFO] [1737514937.542728]: reached home +[INFO] [1737514937.543240]: reset action history +[INFO] [1737514940.257185]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737514941.084885]: resetting environment +[INFO] [1737514941.090766]: cleared current plan +[INFO] [1737514941.090946]: moving to home +[INFO] [1737514942.092590]: reached home +[INFO] [1737514942.093455]: reset action history +[INFO] [1737514944.801916]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/episode_rosbags/episode_1_2025-01-21-22-02-24.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2659946.log +started roslaunch server http://158.130.50.37:36167/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2660421] +process[print_text-5]: started with pid [2660422] +process[rosbag_record-6]: started with pid [2660423] +started bagging! +For topic gripper_width: timestamp difference is 122011870 for nearest: 1737514951912493914 - target: 1737514951790482044 at idx 95 +[INFO] [1737514979.740905]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1737514980.517324]: Stored episode 2. +[INFO] [1737514980.519219]: Saving video... +[INFO] [1737514980.729949]: Video saved! +Starting episode 2 +[INFO] [1737514980.762929]: resetting environment +[INFO] [1737514980.788813]: cleared current plan +[INFO] [1737514980.791754]: moving to home +[INFO] [1737514984.396672]: reached home +[INFO] [1737514984.402245]: reset action history +[INFO] [1737514987.182937]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737514999.068598]: resetting environment +[INFO] [1737514999.076947]: cleared current plan +[INFO] [1737514999.077138]: moving to home +[INFO] [1737515000.078591]: reached home +[INFO] [1737515000.080152]: reset action history +[INFO] [1737515002.849157]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/episode_rosbags/episode_2_2025-01-21-22-03-22.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2659946.log +started roslaunch server http://158.130.50.37:45839/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2660567] +process[print_text-8]: started with pid [2660568] +process[rosbag_record-9]: started with pid [2660569] +started bagging! +For topic gripper_width: timestamp difference is 101847717 for nearest: 1737515005179162332 - target: 1737515005077314615 at idx 33 +For topic gripper_width: timestamp difference is 105739760 for nearest: 1737515009979161191 - target: 1737515009873421431 at idx 94 +[INFO] [1737515039.348670]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1737515040.175489]: Stored episode 3. +[INFO] [1737515040.179175]: Saving video... +[INFO] [1737515040.383474]: Video saved! +Starting episode 3 +[INFO] [1737515040.423240]: resetting environment +[INFO] [1737515040.444867]: cleared current plan +[INFO] [1737515040.445148]: moving to home +[INFO] [1737515044.049677]: reached home +[INFO] [1737515044.051570]: reset action history +[INFO] [1737515046.778780]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737515051.630818]: resetting environment +[INFO] [1737515051.645300]: cleared current plan +[INFO] [1737515051.645499]: moving to home +[INFO] [1737515052.650102]: reached home +[INFO] [1737515052.653477]: reset action history +[INFO] [1737515055.432778]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/episode_rosbags/episode_3_2025-01-21-22-04-15.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2659946.log +started roslaunch server http://158.130.50.37:39565/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2660708] +process[print_text-11]: started with pid [2660709] +process[rosbag_record-12]: started with pid [2660710] +started bagging! +For topic gripper_width: timestamp difference is 112253497 for nearest: 1737515058279136727 - target: 1737515058166883230 at idx 44 +[INFO] [1737515091.130807]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1737515091.839793]: Stored episode 4. +[INFO] [1737515091.845666]: Saving video... +[INFO] [1737515092.098661]: Video saved! +Starting episode 4 +[INFO] [1737515092.180558]: resetting environment +[INFO] [1737515092.226394]: cleared current plan +[INFO] [1737515092.229045]: moving to home +[INFO] [1737515095.845909]: reached home +[INFO] [1737515095.850848]: reset action history +[INFO] [1737515098.645090]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737515100.049220]: resetting environment +[INFO] [1737515100.080107]: cleared current plan +[INFO] [1737515100.084862]: moving to home +[INFO] [1737515101.097219]: reached home +[INFO] [1737515101.103054]: reset action history +[INFO] [1737515103.903374]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/episode_rosbags/episode_4_2025-01-21-22-05-03.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2659946.log +started roslaunch server http://158.130.50.37:36699/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2660848] +process[print_text-14]: started with pid [2660849] +process[rosbag_record-15]: started with pid [2660850] +started bagging! +For topic gripper_width: timestamp difference is 121845868 for nearest: 1737515112579139140 - target: 1737515112457293272 at idx 123 +[INFO] [1737515141.445044]: Storing episode... +[rosbag_record-15] killing on exit[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1737515142.452509]: Stored episode 5. +[INFO] [1737515142.453359]: Saving video... +[INFO] [1737515142.730489]: Video saved! +[INFO] [1737515142.793192]: resetting environment +[WARN] [1737515142.768667]: Plan exhausted +[WARN] [1737515142.808899]: Plan exhausted +[INFO] [1737515142.836891]: cleared current plan +[INFO] [1737515142.837148]: moving to home +[INFO] [1737515146.440636]: reached home +[INFO] [1737515146.447591]: reset action history +[INFO] [1737515149.236036]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/requirements.txt b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-metadata.json b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..cfa122e1f8d272fe027f20cf6dafcab2395bf381 --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T03:01:10.590028", + "startedAt": "2025-01-22T03:01:10.206489", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3747.7392499999996, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.155, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 437.4647331237793 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..94181c74b40b8e3a75daf9331b956854d316f0d0 --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "f0d20ac03e2b6a636fd4b75dc420d38bca78737161b03b456f4bafb0c9d3ac5c", "size": 1482819, "path": "media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4"}, "global_step": 665, "_timestamp": 1737515167.7197227, "_runtime": 297.50294280052185, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "c351cf40eb1c20bd3656095d6296b33b5c1d6813b7dc6d200beb36a1247d8eb4", "size": 1501132, "path": "media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4"}, "eval/num_success": 5.0, "episode": 15.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "977e432fb56e9f58fb599098d19508aadfcf98861d2346a062f799302713a2ce", "size": 1562026, "path": "media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "d1e75bdfd0a4e32aff60e01308eeaf7bec80ee9a3c08c90f23113a3ee4e9300a", "size": 1508395, "path": "media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "ac7a1d089ff431aae59a91f891770d23cf42ae6990d7ccf3519ffdfae0139d56", "size": 1615658, "path": "media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://5bc28s92fqhktl919l1m84x8j2i8joco0opuh32hrru2y2255ddbhk7zmoiicd7hv0ff5wm3e9a3oakoy1k8byzynr45tef1y3wqyj1uejqj5op5cy9fx144io2w5tvs/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://45bgahz4ppr3a0mwjn8bkqdccd8h1nsqnhddnzvhe0xdfu0llj7n96vcjsxl0pmyvbt09s7higdbdp26hohgel8v59k5fa3ktqmdi6ikbojzo9p6wrie08v6bor6xfny:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 15, "total_num_episodes": 15, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9928352235762072, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.7993280141947043, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.8249338715618673, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "6e312070ee781a67b6ed562ed805cc283e04b6daa3f3dedf9db9ab8790e00e5b", "size": 420, "artifact_path": "wandb-client-artifact://flz7ndmewls6bpz7wotesst3c007rlc6wb1gpqdkmw2lbav3at4nxcj00cpzkya0t8q5xcc3uyh7rvy6acrbmvar2wlkp51tdxmpdz5kpktugj1igqqi6gc7bhcmjire/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ejsgywdsdrkhsoml7ye8t6nomd7nn5vzg58shutczrzqcoy77cagn7cswfkdt6je6jti5tiy4p3bbutvjiccpolhid9hj56me9fatibep4ox1x0pyqyxscydvhq1p4x5:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 330}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9928352236747742, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.7993280291557312, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.8249338865280151, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/logs/debug-internal.log b/4lg2e533/wandb/run-20250121_220110-4lg2e533/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f41bff36ef289abf94e47b78b5853b4058df7706 --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/logs/debug-internal.log @@ -0,0 +1,858 @@ +2025-01-21 22:01:10,215 INFO StreamThr :2660143 [internal.py:wandb_internal():85] W&B internal server running at pid: 2660143, started at: 2025-01-21 22:01:10.215539 +2025-01-21 22:01:10,216 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status +2025-01-21 22:01:10,232 INFO WriterThread:2660143 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/run-4lg2e533.wandb +2025-01-21 22:01:10,233 DEBUG SenderThread:2660143 [sender.py:send():379] send: header +2025-01-21 22:01:10,240 DEBUG SenderThread:2660143 [sender.py:send():379] send: run +2025-01-21 22:01:10,481 INFO SenderThread:2660143 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files +2025-01-21 22:01:10,481 INFO SenderThread:2660143 [sender.py:_start_run_threads():1188] run started: 4lg2e533 with start time 1737514870.21678 +2025-01-21 22:01:10,496 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 22:01:10,496 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: check_version +2025-01-21 22:01:10,570 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 22:01:10,575 DEBUG HandlerThread:2660143 [system_info.py:__init__():26] System info init +2025-01-21 22:01:10,575 DEBUG HandlerThread:2660143 [system_info.py:__init__():41] System info init done +2025-01-21 22:01:10,575 INFO HandlerThread:2660143 [system_monitor.py:start():194] Starting system monitor +2025-01-21 22:01:10,575 INFO SystemMonitor:2660143 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 22:01:10,576 INFO HandlerThread:2660143 [system_monitor.py:probe():214] Collecting system info +2025-01-21 22:01:10,576 INFO SystemMonitor:2660143 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 22:01:10,577 INFO SystemMonitor:2660143 [interfaces.py:start():188] Started disk monitoring +2025-01-21 22:01:10,577 INFO SystemMonitor:2660143 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 22:01:10,579 INFO SystemMonitor:2660143 [interfaces.py:start():188] Started memory monitoring +2025-01-21 22:01:10,579 INFO SystemMonitor:2660143 [interfaces.py:start():188] Started network monitoring +2025-01-21 22:01:10,589 DEBUG HandlerThread:2660143 [system_info.py:probe():152] Probing system +2025-01-21 22:01:10,591 DEBUG HandlerThread:2660143 [system_info.py:_probe_git():137] Probing git +2025-01-21 22:01:10,596 DEBUG HandlerThread:2660143 [system_info.py:_probe_git():145] Probing git done +2025-01-21 22:01:10,596 DEBUG HandlerThread:2660143 [system_info.py:probe():200] Probing system done +2025-01-21 22:01:10,596 DEBUG HandlerThread:2660143 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T03:01:10.590028', 'startedAt': '2025-01-22T03:01:10.206489', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3747.7392499999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.155, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 437.4647331237793}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 22:01:10,596 INFO HandlerThread:2660143 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 22:01:10,596 INFO HandlerThread:2660143 [system_monitor.py:probe():227] Publishing system info +2025-01-21 22:01:10,596 DEBUG HandlerThread:2660143 [system_info.py:_save_code():44] Saving code +2025-01-21 22:01:10,597 DEBUG HandlerThread:2660143 [system_info.py:_save_code():67] Saving code done +2025-01-21 22:01:10,597 DEBUG HandlerThread:2660143 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 22:01:10,659 DEBUG HandlerThread:2660143 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 22:01:10,660 INFO HandlerThread:2660143 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 22:01:10,663 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:01:10,663 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 22:01:10,663 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 22:01:10,664 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 22:01:10,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 22:01:10,919 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/vrmadxke-wandb-metadata.json +2025-01-21 22:01:10,919 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: python_packages +2025-01-21 22:01:10,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:01:10,920 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:01:10,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:10,924 INFO HandlerThread:2660143 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/tb" + save: true +} +control { + relay_id: "fbf885f55bfb41ae9fd24d9eb85e2419" +} +_info { + stream_id: "4lg2e533" +} + +2025-01-21 22:01:10,924 DEBUG HandlerThread:2660143 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 22:01:10,938 INFO wandb-upload_2:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/fpn7u3em-diff.patch +2025-01-21 22:01:10,950 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/2hautu9g-code/FISH/eval_robot.py +2025-01-21 22:01:11,009 DEBUG SenderThread:2660143 [sender.py:send():379] send: telemetry +2025-01-21 22:01:11,009 DEBUG SenderThread:2660143 [sender.py:send():379] send: tbrecord +2025-01-21 22:01:11,010 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:01:11,010 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 with policy live +2025-01-21 22:01:11,202 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/1n40o3p2-events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:01:11,482 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/diff.patch +2025-01-21 22:01:11,483 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/requirements.txt +2025-01-21 22:01:11,483 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:01:11,483 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-metadata.json +2025-01-21 22:01:11,483 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:11,483 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/code/FISH/eval_robot.py +2025-01-21 22:01:11,483 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/code +2025-01-21 22:01:11,483 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/code/FISH +2025-01-21 22:01:11,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:12,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:13,482 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:13,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:14,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:15,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:15,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:16,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:17,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:18,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:19,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:20,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:20,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:20,975 DEBUG SenderThread:2660143 [sender.py:send():379] send: config +2025-01-21 22:01:21,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:22,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:23,484 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:23,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:24,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:25,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:01:25,919 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:01:25,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:26,070 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:26,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:27,486 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:27,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:28,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:29,486 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:29,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:30,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:31,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:31,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:32,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:33,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:34,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:35,488 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:35,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:36,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:37,489 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:37,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:37,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:38,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:39,489 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:39,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:40,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:01:40,919 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:01:40,961 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:41,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:42,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:43,099 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:43,490 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/config.yaml +2025-01-21 22:01:43,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:44,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:45,491 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:45,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:46,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:47,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:48,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:48,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:49,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:50,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:51,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:52,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:53,493 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:01:53,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:54,889 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:01:54,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:55,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:01:55,919 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:01:55,961 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:56,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:57,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:58,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:59,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:01:59,982 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:00,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:01,495 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:01,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:02,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:03,496 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:03,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:04,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:05,425 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:05,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:06,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:07,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:08,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:09,927 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:10,579 DEBUG SystemMonitor:2660143 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 22:02:10,580 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:02:10,580 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:10,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:02:10,919 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:02:10,961 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:11,928 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:12,928 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:13,922 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:02:13,922 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:02:13,922 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 with policy now +2025-01-21 22:02:13,923 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:02:13,923 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:02:13,924 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:02:13,929 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:14,171 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/qoja1q6g-media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 +2025-01-21 22:02:14,499 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:02:14,499 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 +2025-01-21 22:02:14,499 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos +2025-01-21 22:02:14,499 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media +2025-01-21 22:02:14,499 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval +2025-01-21 22:02:14,791 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/hwuybd8d-media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 +2025-01-21 22:02:14,928 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:15,499 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:15,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:15,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:16,928 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:17,929 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:18,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:19,501 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:19,929 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:20,929 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:21,070 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:21,501 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:21,501 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:02:21,929 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:22,165 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/cyezw7ny-events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:02:22,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:23,502 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:23,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:24,931 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:25,919 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:02:25,920 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:02:25,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:26,085 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:26,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:27,503 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:27,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:28,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:29,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:30,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:31,916 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:31,961 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:32,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:33,505 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:02:33,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:34,930 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:35,931 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:36,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:36,932 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:37,932 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:38,933 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:39,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:40,580 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:02:40,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:02:40,920 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:02:40,965 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:41,933 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:42,062 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:42,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:43,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:44,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:45,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:46,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:47,922 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:47,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:48,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:49,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:50,935 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:51,936 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:52,922 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:52,935 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:53,936 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:54,936 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:55,920 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:02:55,920 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:02:55,961 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:56,936 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:57,936 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:58,074 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:02:58,936 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:02:59,945 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:00,760 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:03:00,760 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 with policy now +2025-01-21 22:03:00,760 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:03:00,760 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:03:00,761 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:03:00,778 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:03:00,937 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:00,999 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/4wkmge9q-media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 +2025-01-21 22:03:01,515 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:03:01,515 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:01,515 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 +2025-01-21 22:03:01,515 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval +2025-01-21 22:03:01,816 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/m3054ytg-media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 +2025-01-21 22:03:01,938 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:02,940 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:03,516 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:03,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:03,945 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:04,943 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:05,517 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:05,942 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:06,945 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:07,948 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:08,948 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:09,225 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:09,518 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:09,956 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:10,581 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:03:10,923 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:03:10,923 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:03:10,969 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:11,949 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:12,949 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:13,953 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:14,951 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:15,068 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:15,953 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:16,954 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:17,954 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:18,953 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:19,356 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:03:19,356 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:03:19,357 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:03:19,521 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:03:19,521 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:03:19,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:20,079 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:20,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:21,522 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:21,960 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:22,959 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:23,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:24,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:25,190 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:25,523 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:25,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:03:25,921 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:03:25,954 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:26,954 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:27,525 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:27,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:28,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:29,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:30,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:30,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:31,526 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:03:31,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:32,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:33,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:34,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:35,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:35,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:36,955 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:37,956 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:38,956 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:39,956 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:40,581 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:03:40,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:03:40,921 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:03:40,961 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:41,070 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:41,956 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:42,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:43,956 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:44,956 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:45,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:46,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:46,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:47,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:48,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:49,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:50,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:51,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:51,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:52,958 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:53,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:54,957 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:55,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:03:55,921 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:03:55,965 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:56,958 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:57,056 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:03:57,958 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:58,958 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:03:59,962 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:00,410 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:04:00,410 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 with policy now +2025-01-21 22:04:00,413 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:04:00,414 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:04:00,414 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:04:00,427 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:04:00,537 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:04:00,537 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 +2025-01-21 22:04:00,538 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval +2025-01-21 22:04:00,658 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/y0ky4200-media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 +2025-01-21 22:04:00,967 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:00,976 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/vnq1vyz5-media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 +2025-01-21 22:04:01,538 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:01,962 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:02,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:02,969 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:03,965 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:04,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:05,539 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:05,968 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:06,965 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:07,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:08,825 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:08,970 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:09,541 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:09,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:10,582 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:04:10,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:04:10,926 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:04:10,976 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:11,969 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:12,200 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:04:12,200 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:04:12,202 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:04:12,542 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:04:12,542 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:04:12,809 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/q9f1eby7-events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:04:12,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:13,542 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:13,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:14,693 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:14,978 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:15,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:16,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:17,544 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:17,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:18,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:19,544 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:19,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:20,329 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:20,971 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:21,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:22,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:23,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:24,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:25,921 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:04:25,922 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:04:25,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:26,010 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:26,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:27,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:28,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:29,973 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:30,972 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:31,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:31,975 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:32,973 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:33,973 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:34,974 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:35,974 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:36,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:36,974 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:37,974 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:38,978 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:39,975 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:40,582 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:04:40,922 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:04:40,922 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:04:40,975 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:41,975 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:41,985 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:42,976 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:43,975 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:44,975 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:45,975 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:46,976 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:47,928 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:47,976 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:48,976 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:49,976 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:50,976 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:51,979 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:52,143 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:04:52,143 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 with policy now +2025-01-21 22:04:52,155 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:04:52,156 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:04:52,156 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:04:52,388 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:04:52,388 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/zdqszyn5-media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 +2025-01-21 22:04:52,556 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:04:52,556 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 +2025-01-21 22:04:52,556 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval +2025-01-21 22:04:52,851 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/1i68o5ah-media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 +2025-01-21 22:04:52,992 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:53,407 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:53,556 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:53,989 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:54,986 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:55,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:04:55,925 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:04:55,988 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:56,985 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:57,557 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:57,988 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:58,651 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:04:58,983 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:04:59,558 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:04:59,989 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:00,262 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:05:00,262 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:05:00,263 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:05:00,558 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:05:00,558 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:05:01,003 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:01,559 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:01,991 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:02,996 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:03,559 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:03,903 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:03,993 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:04,996 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:05,560 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:05,990 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:06,991 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:07,561 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:07,990 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:08,991 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:09,038 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:09,561 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:09,991 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:10,583 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:05:10,922 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:05:10,923 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:05:10,991 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:11,991 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:12,992 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:13,563 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:13,997 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:14,629 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:14,996 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:15,994 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:16,997 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:17,994 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:18,994 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:19,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:19,995 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:20,995 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:21,994 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:22,995 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:23,995 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:24,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:24,996 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:25,922 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:05:25,922 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:05:26,002 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:27,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:27,999 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:29,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:30,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:30,076 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:31,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:32,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:33,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:34,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:35,001 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:35,935 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:36,000 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:37,001 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:38,002 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:39,002 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:40,001 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:40,583 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:05:40,924 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:05:40,924 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:05:41,001 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:41,039 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:42,012 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:42,787 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:05:42,787 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 with policy now +2025-01-21 22:05:42,793 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:05:42,794 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:05:42,794 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:05:42,800 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:05:43,002 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:43,094 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/3hf7alg0-media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 +2025-01-21 22:05:43,573 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:05:43,573 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:43,573 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 +2025-01-21 22:05:43,573 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval +2025-01-21 22:05:43,888 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/8ee5ptx9-media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 +2025-01-21 22:05:44,015 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:45,007 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:45,574 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:46,015 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:46,441 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:47,012 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:47,574 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:48,014 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:49,014 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:50,014 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:51,013 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:51,576 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:05:51,945 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:52,017 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:53,016 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:54,013 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:55,013 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:55,934 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:05:55,934 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:05:56,023 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:57,021 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:57,073 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:05:58,016 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:05:59,018 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:00,017 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:01,026 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:02,022 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:02,953 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:03,030 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:04,021 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:05,020 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:06,029 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:06,340 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:06:06,341 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:06,342 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:06,581 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:06:06,581 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:06:06,832 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/j3cwuhh8-events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:06:07,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:07,283 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:06:07,284 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:06:07,445 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:06:07,581 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:06:07,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:06:07,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media +2025-01-21 22:06:07,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table +2025-01-21 22:06:07,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval +2025-01-21 22:06:07,762 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/yducq79t-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:06:07,845 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:06:07,855 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:06:07,880 INFO wandb-upload_1:2660143 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp7ay50xjm +2025-01-21 22:06:08,024 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:08,359 INFO SenderThread:2660143 [sender.py:send_request_log_artifact():1525] logged artifact run-4lg2e533-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcxNTI2NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwMzU1OA==', 'latestArtifact': None}} +2025-01-21 22:06:08,359 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:06:08,359 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-21 22:06:08,359 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:06:08,359 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,360 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:08,360 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,361 DEBUG SenderThread:2660143 [sender.py:send():379] send: telemetry +2025-01-21 22:06:08,361 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,365 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,365 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,366 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,366 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,366 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,366 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,367 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,367 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,367 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,367 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,367 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,368 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,368 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,368 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,368 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,368 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,369 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,369 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,369 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,369 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:08,370 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:08,370 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:06:08,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:06:08,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:06:08,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-21 22:06:08,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval +2025-01-21 22:06:08,621 INFO wandb-upload_2:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/143fr4um-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:06:08,785 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/wrrc4rhk-media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-21 22:06:09,027 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:09,582 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:06:09,584 INFO wandb-upload_0:2660143 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgkkabjf_ +2025-01-21 22:06:10,024 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:10,433 INFO SenderThread:2660143 [sender.py:send_request_log_artifact():1525] logged artifact run-4lg2e533-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcxNTI4Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwMzU2MA==', 'latestArtifact': None}} +2025-01-21 22:06:10,433 DEBUG SenderThread:2660143 [sender.py:send():379] send: files +2025-01-21 22:06:10,433 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json with policy now +2025-01-21 22:06:10,434 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:06:10,434 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:10,435 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:10,583 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:06:10,584 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:06:10,678 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/5dlufjw4-media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-21 22:06:10,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:06:10,925 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:06:11,024 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:12,026 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:13,025 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:14,012 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:14,025 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:15,025 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:16,025 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:17,026 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:18,026 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:19,030 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:19,945 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:20,027 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:21,027 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:22,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:23,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:24,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:24,945 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:25,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:25,926 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:06:25,926 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:06:26,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:27,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:28,028 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:29,029 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:30,030 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:30,077 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:31,032 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:32,029 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:33,029 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:34,032 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:35,030 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:35,948 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:36,046 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:36,593 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/config.yaml +2025-01-21 22:06:37,031 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:38,031 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:39,031 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:40,031 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:40,584 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:06:40,925 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:06:40,926 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:06:41,033 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:41,244 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:41,517 DEBUG SenderThread:2660143 [sender.py:send():379] send: exit +2025-01-21 22:06:41,517 INFO SenderThread:2660143 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 22:06:41,518 INFO SenderThread:2660143 [sender.py:send_exit():588] handling runtime: 330 +2025-01-21 22:06:41,518 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:41,518 INFO SenderThread:2660143 [sender.py:send_exit():594] send defer +2025-01-21 22:06:41,518 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:41,518 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 22:06:41,518 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:41,518 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 22:06:41,518 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 1 +2025-01-21 22:06:41,518 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:41,519 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 22:06:41,519 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:41,519 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 22:06:41,519 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 2 +2025-01-21 22:06:41,519 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:41,519 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 22:06:41,519 INFO HandlerThread:2660143 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 22:06:41,519 DEBUG SystemMonitor:2660143 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 22:06:41,519 DEBUG SystemMonitor:2660143 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 22:06:41,519 INFO HandlerThread:2660143 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 22:06:41,520 INFO HandlerThread:2660143 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 22:06:41,571 INFO HandlerThread:2660143 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 22:06:41,571 INFO HandlerThread:2660143 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 22:06:41,571 INFO HandlerThread:2660143 [interfaces.py:finish():200] Joined network monitor +2025-01-21 22:06:41,572 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:41,572 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 22:06:41,572 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 3 +2025-01-21 22:06:41,572 DEBUG SenderThread:2660143 [sender.py:send():379] send: stats +2025-01-21 22:06:41,572 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:41,572 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 22:06:41,573 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:41,573 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 22:06:41,573 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 4 +2025-01-21 22:06:41,573 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:41,573 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 22:06:41,595 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:06:43,596 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:06:48,352 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:06:48,353 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:48,353 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:48,353 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 22:06:48,353 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 5 +2025-01-21 22:06:48,353 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 22:06:48,353 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:06:48,354 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:48,354 DEBUG SenderThread:2660143 [sender.py:send():379] send: history +2025-01-21 22:06:48,354 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 22:06:48,355 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:06:48,355 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:48,356 DEBUG SenderThread:2660143 [sender.py:send():379] send: summary +2025-01-21 22:06:48,360 INFO SenderThread:2660143 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:06:48,360 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:48,360 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 22:06:48,360 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 6 +2025-01-21 22:06:48,360 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:48,360 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 22:06:48,360 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:48,360 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 22:06:48,360 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 7 +2025-01-21 22:06:48,360 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:06:48,361 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:48,361 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 22:06:48,361 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:48,361 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 22:06:48,519 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:06:48,598 INFO Thread-12 :2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:06:50,837 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 8 +2025-01-21 22:06:50,837 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:06:50,837 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:50,838 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 22:06:50,838 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:50,838 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 22:06:50,838 INFO SenderThread:2660143 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 22:06:50,841 INFO SenderThread:2660143 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 22:06:50,843 INFO SenderThread:2660143 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 22:06:50,856 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 9 +2025-01-21 22:06:50,857 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:50,857 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 22:06:50,857 DEBUG SenderThread:2660143 [sender.py:send():379] send: artifact +2025-01-21 22:06:51,136 INFO SenderThread:2660143 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3MzQyMw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 22:06:51,136 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:51,136 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 22:06:51,136 INFO SenderThread:2660143 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 22:06:51,520 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:06:51,599 INFO SenderThread:2660143 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:06:51,599 INFO SenderThread:2660143 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files +2025-01-21 22:06:51,599 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/config.yaml config.yaml +2025-01-21 22:06:51,599 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json wandb-summary.json +2025-01-21 22:06:51,599 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/requirements.txt requirements.txt +2025-01-21 22:06:51,599 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-metadata.json wandb-metadata.json +2025-01-21 22:06:51,600 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:06:51,603 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log output.log +2025-01-21 22:06:51,605 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/diff.patch diff.patch +2025-01-21 22:06:51,607 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 22:06:51,608 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 media/videos/eval/3_eval_3_d1e75bdfd0a4e32aff60.mp4 +2025-01-21 22:06:51,608 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 media/videos/eval/1_eval_1_c351cf40eb1c20bd3656.mp4 +2025-01-21 22:06:51,608 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 media/videos/eval/0_eval_0_f0d20ac03e2b6a636fd4.mp4 +2025-01-21 22:06:51,609 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 media/videos/eval/2_eval_2_977e432fb56e9f58fb59.mp4 +2025-01-21 22:06:51,609 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 media/videos/eval/4_eval_4_ac7a1d089ff431aae59a.mp4 +2025-01-21 22:06:51,609 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-21 22:06:51,609 INFO SenderThread:2660143 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:06:51,609 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 10 +2025-01-21 22:06:51,609 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:06:51,609 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:51,609 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 22:06:51,610 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:51,610 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 22:06:51,610 INFO SenderThread:2660143 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:06:51,780 INFO wandb-upload_0:2660143 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/config.yaml +2025-01-21 22:06:51,818 INFO wandb-upload_2:2660143 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/requirements.txt +2025-01-21 22:06:51,839 INFO wandb-upload_4:2660143 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/output.log +2025-01-21 22:06:51,839 INFO wandb-upload_1:2660143 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/files/wandb-summary.json +2025-01-21 22:06:51,910 INFO wandb-upload_3:2660143 [upload_job.py:push():130] Uploaded file /tmp/tmpo2hqivb2wandb/fosgc2nr-events.out.tfevents.1737514870.leonmkim-ROG-Strix-G15CS-G15CS.2659946.0 +2025-01-21 22:06:52,110 INFO Thread-11 (_thread_body):2660143 [sender.py:transition_state():614] send defer: 11 +2025-01-21 22:06:52,110 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:52,110 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 22:06:52,110 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:52,111 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 22:06:52,111 INFO SenderThread:2660143 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 22:06:52,112 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 12 +2025-01-21 22:06:52,112 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:52,112 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 22:06:52,112 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:52,112 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 22:06:52,112 INFO SenderThread:2660143 [file_stream.py:finish():601] file stream finish called +2025-01-21 22:06:52,322 INFO SenderThread:2660143 [file_stream.py:finish():605] file stream finish is done +2025-01-21 22:06:52,322 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 13 +2025-01-21 22:06:52,323 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:52,323 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 22:06:52,323 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:52,323 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 22:06:52,323 INFO SenderThread:2660143 [sender.py:transition_state():614] send defer: 14 +2025-01-21 22:06:52,323 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:06:52,323 INFO HandlerThread:2660143 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 22:06:52,324 DEBUG SenderThread:2660143 [sender.py:send():379] send: final +2025-01-21 22:06:52,324 DEBUG SenderThread:2660143 [sender.py:send():379] send: footer +2025-01-21 22:06:52,324 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: defer +2025-01-21 22:06:52,324 INFO SenderThread:2660143 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 22:06:52,324 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:06:52,324 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:06:52,325 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 22:06:52,325 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: server_info +2025-01-21 22:06:52,326 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:06:52,327 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 22:06:52,327 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 22:06:52,328 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:06:52,386 DEBUG SenderThread:2660143 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:06:52,386 INFO MainThread:2660143 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 22:06:52,387 INFO MainThread:2660143 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 22:06:52,387 INFO MainThread:2660143 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 22:06:52,387 DEBUG HandlerThread:2660143 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 22:06:52,387 INFO HandlerThread:2660143 [handler.py:finish():882] shutting down handler +2025-01-21 22:06:53,327 INFO WriterThread:2660143 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/run-4lg2e533.wandb +2025-01-21 22:06:53,386 INFO SenderThread:2660143 [sender.py:finish():1615] shutting down sender +2025-01-21 22:06:53,387 INFO SenderThread:2660143 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:06:53,387 INFO SenderThread:2660143 [file_pusher.py:join():175] waiting for file pusher diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/logs/debug.log b/4lg2e533/wandb/run-20250121_220110-4lg2e533/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d5ff3cd5d214c7be4fb3fac7136c8156bf0885f8 --- /dev/null +++ b/4lg2e533/wandb/run-20250121_220110-4lg2e533/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Configure stats pid to 2659946 +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/settings +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/logs/debug.log +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/wandb/run-20250121_220110-4lg2e533/logs/debug-internal.log +2025-01-21 22:01:10,208 INFO MainThread:2659946 [wandb_init.py:init():569] calling init triggers +2025-01-21 22:01:10,209 INFO MainThread:2659946 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_1', 'wandb_notes': '4295_1_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533'} +2025-01-21 22:01:10,209 INFO MainThread:2659946 [wandb_init.py:init():619] starting backend +2025-01-21 22:01:10,209 INFO MainThread:2659946 [wandb_init.py:init():623] setting up manager +2025-01-21 22:01:10,214 INFO MainThread:2659946 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 22:01:10,216 INFO MainThread:2659946 [wandb_init.py:init():631] backend started and connected +2025-01-21 22:01:10,232 INFO MainThread:2659946 [wandb_init.py:init():720] updated telemetry +2025-01-21 22:01:10,238 INFO MainThread:2659946 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 22:01:10,495 INFO MainThread:2659946 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 22:01:10,566 INFO MainThread:2659946 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 22:01:10,567 INFO MainThread:2659946 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 22:01:10,919 INFO MainThread:2659946 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 22:01:10,920 INFO MainThread:2659946 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 22:01:10,921 INFO MainThread:2659946 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 22:01:10,921 INFO MainThread:2659946 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 22:01:10,923 INFO MainThread:2659946 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 22:01:10,924 INFO MainThread:2659946 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/4lg2e533/tb, True +2025-01-21 22:01:20,975 INFO MainThread:2659946 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 22:06:53,388 WARNING MsgRouterThr:2659946 [router.py:message_loop():77] message_loop has been closed diff --git a/4lg2e533/wandb/run-20250121_220110-4lg2e533/run-4lg2e533.wandb b/4lg2e533/wandb/run-20250121_220110-4lg2e533/run-4lg2e533.wandb new file mode 100644 index 0000000000000000000000000000000000000000..8373292da1c8aba1fba1ac63296e7279df433e10 Binary files /dev/null and b/4lg2e533/wandb/run-20250121_220110-4lg2e533/run-4lg2e533.wandb differ diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43b4c6921018a9c64142bb23198358c4db31d01f --- /dev/null +++ b/config.yaml @@ -0,0 +1,575 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 1 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '4295_1' +wandb_notes: 4295_1_restarted_1 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: false +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: true +use_sdf_maps: true +use_normals_maps: true +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 7 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +restore_missing_checkpoint: null +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ${test.root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${test.name_of_expert_demo} + expert_dataset: ${test.expert_dataset_dirpath}/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ${test.expert_dataset_dirpath}/${test.semantic_demo_grouping_name} + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: ${test.local_snapshot_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name} + resume_wandb_run: false +feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/kopteun2/.hydra/config.yaml b/kopteun2/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2154a6acb64e2809a71d0adfc508bd71e0da7dde --- /dev/null +++ b/kopteun2/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/kopteun2/.hydra/hydra.yaml b/kopteun2/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c642699bf93f5890b2987873fde1bc899b4487c0 --- /dev/null +++ b/kopteun2/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/kopteun2 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/kopteun2/.hydra/overrides.yaml b/kopteun2/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/kopteun2/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/kopteun2/eval_policy.log b/kopteun2/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..cf55aa0c1cbcd79e13d656c01a18fd5c6563a591 --- /dev/null +++ b/kopteun2/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-21 22:01:02,017][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 22:01:02,020][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-21 22:01:05,362][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/nf7i3zuf/episode_rosbags/aligned_depth_to_color_K.npy b/nf7i3zuf/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/nf7i3zuf/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/nf7i3zuf/episode_rosbags/cam_tf_world.npy b/nf7i3zuf/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/nf7i3zuf/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/nf7i3zuf/episode_rosbags/depth_K.npy b/nf7i3zuf/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/nf7i3zuf/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/nf7i3zuf/episode_rosbags/episode_1_2025-01-21-21-46-18.bag b/nf7i3zuf/episode_rosbags/episode_1_2025-01-21-21-46-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..d4262bd883429ecbae4d3534f42e4b1c05796318 --- /dev/null +++ b/nf7i3zuf/episode_rosbags/episode_1_2025-01-21-21-46-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f8fcac81a32c4c752f14c67bab7b33c5b6d42918da54a613d490b9981679297 +size 2598722156 diff --git a/nf7i3zuf/episode_rosbags/episode_3_2025-01-21-21-47-59.bag b/nf7i3zuf/episode_rosbags/episode_3_2025-01-21-21-47-59.bag new file mode 100644 index 0000000000000000000000000000000000000000..b15dfb55b532d2fb8a3d4742ac63d1732aabe104 --- /dev/null +++ b/nf7i3zuf/episode_rosbags/episode_3_2025-01-21-21-47-59.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f09708427d517b71d50b729af19bb3a9bea2d59930cd865b0d19dfb911fc9df +size 2593112709 diff --git a/nf7i3zuf/eval_robot.log b/nf7i3zuf/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..2038b315f6e7bad49ff6a948ffdb1baf1384d074 --- /dev/null +++ b/nf7i3zuf/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 21:45:02,656][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:45:02,659][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/nf7i3zuf/wandb/debug-internal.log b/nf7i3zuf/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5c18e83b00341ac3a16fd6de988db4cdb0c597b9 --- /dev/null +++ b/nf7i3zuf/wandb/debug-internal.log @@ -0,0 +1,808 @@ +2025-01-21 21:45:08,624 INFO StreamThr :2655921 [internal.py:wandb_internal():85] W&B internal server running at pid: 2655921, started at: 2025-01-21 21:45:08.623939 +2025-01-21 21:45:08,625 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:45:08,636 INFO WriterThread:2655921 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/run-nf7i3zuf.wandb +2025-01-21 21:45:08,636 DEBUG SenderThread:2655921 [sender.py:send():379] send: header +2025-01-21 21:45:08,644 DEBUG SenderThread:2655921 [sender.py:send():379] send: run +2025-01-21 21:45:08,920 INFO SenderThread:2655921 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files +2025-01-21 21:45:08,920 INFO SenderThread:2655921 [sender.py:_start_run_threads():1188] run started: nf7i3zuf with start time 1737513908.624247 +2025-01-21 21:45:08,936 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:45:08,936 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:45:09,006 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:45:09,011 DEBUG HandlerThread:2655921 [system_info.py:__init__():26] System info init +2025-01-21 21:45:09,011 DEBUG HandlerThread:2655921 [system_info.py:__init__():41] System info init done +2025-01-21 21:45:09,011 INFO HandlerThread:2655921 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:45:09,011 INFO SystemMonitor:2655921 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:45:09,011 INFO HandlerThread:2655921 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:45:09,012 INFO SystemMonitor:2655921 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:45:09,013 INFO SystemMonitor:2655921 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:45:09,013 INFO SystemMonitor:2655921 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:45:09,014 INFO SystemMonitor:2655921 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:45:09,014 INFO SystemMonitor:2655921 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:45:09,022 DEBUG HandlerThread:2655921 [system_info.py:probe():152] Probing system +2025-01-21 21:45:09,024 DEBUG HandlerThread:2655921 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:45:09,029 DEBUG HandlerThread:2655921 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:45:09,029 DEBUG HandlerThread:2655921 [system_info.py:probe():200] Probing system done +2025-01-21 21:45:09,029 DEBUG HandlerThread:2655921 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:45:09.022974', 'startedAt': '2025-01-22T02:45:08.614701', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3742.508625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4491.531, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.014, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 406.1821403503418}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:45:09,029 INFO HandlerThread:2655921 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:45:09,030 INFO HandlerThread:2655921 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:45:09,030 DEBUG HandlerThread:2655921 [system_info.py:_save_code():44] Saving code +2025-01-21 21:45:09,031 DEBUG HandlerThread:2655921 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:45:09,031 DEBUG HandlerThread:2655921 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:45:09,094 DEBUG HandlerThread:2655921 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:45:09,094 INFO HandlerThread:2655921 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:45:09,097 DEBUG SenderThread:2655921 [sender.py:send():379] send: files +2025-01-21 21:45:09,098 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:45:09,098 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:45:09,098 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:45:09,329 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:45:09,329 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:45:09,329 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:45:09,330 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:45:09,331 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:09,331 INFO HandlerThread:2655921 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/tb" + save: true +} +control { + relay_id: "a53a85b8e55e4234be073221a47a1b23" +} +_info { + stream_id: "nf7i3zuf" +} + +2025-01-21 21:45:09,331 DEBUG HandlerThread:2655921 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:45:09,356 INFO wandb-upload_0:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/mmdn0nsn-wandb-metadata.json +2025-01-21 21:45:09,365 INFO wandb-upload_1:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/32mtirnp-code/FISH/eval_robot.py +2025-01-21 21:45:09,366 INFO wandb-upload_2:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/uj6dwg7z-diff.patch +2025-01-21 21:45:09,444 DEBUG SenderThread:2655921 [sender.py:send():379] send: telemetry +2025-01-21 21:45:09,444 DEBUG SenderThread:2655921 [sender.py:send():379] send: tbrecord +2025-01-21 21:45:09,444 DEBUG SenderThread:2655921 [sender.py:send():379] send: files +2025-01-21 21:45:09,445 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file events.out.tfevents.1737513909.leonmkim-ROG-Strix-G15CS-G15CS.2655690.0 with policy live +2025-01-21 21:45:09,654 INFO wandb-upload_0:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/f7gat0n6-events.out.tfevents.1737513909.leonmkim-ROG-Strix-G15CS-G15CS.2655690.0 +2025-01-21 21:45:09,922 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:45:09,922 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-metadata.json +2025-01-21 21:45:09,922 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/code/FISH/eval_robot.py +2025-01-21 21:45:09,922 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/diff.patch +2025-01-21 21:45:09,923 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/events.out.tfevents.1737513909.leonmkim-ROG-Strix-G15CS-G15CS.2655690.0 +2025-01-21 21:45:09,923 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/requirements.txt +2025-01-21 21:45:09,923 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/code/FISH +2025-01-21 21:45:09,923 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/code +2025-01-21 21:45:10,330 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:11,330 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:11,921 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:45:12,330 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:13,330 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:14,330 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:45:14,330 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:15,324 DEBUG SenderThread:2655921 [sender.py:send():379] send: config +2025-01-21 21:45:15,365 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:16,331 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:17,331 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:17,923 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:45:18,331 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:19,330 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:45:19,331 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:20,331 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:21,331 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:21,924 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:45:22,331 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:23,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:23,925 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:45:24,329 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:45:24,329 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:45:24,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:24,405 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:45:25,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:25,925 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:45:26,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:27,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:27,926 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:45:28,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:29,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:29,926 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:45:30,236 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:45:30,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:31,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:31,927 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:45:32,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:33,333 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:34,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:35,333 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:36,125 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:45:36,333 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:37,333 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:37,929 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:45:38,333 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:39,329 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:45:39,329 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:45:39,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:40,334 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:41,334 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:41,524 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:45:41,930 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:45:41,930 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/config.yaml +2025-01-21 21:45:42,334 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:43,334 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:44,334 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:45,334 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:46,335 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:47,331 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:45:47,335 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:48,335 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:49,335 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:50,335 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:51,335 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:52,335 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:53,331 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:45:53,335 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:54,329 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:45:54,329 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:45:54,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:55,335 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:56,336 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:57,336 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:58,336 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:45:58,416 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:45:59,337 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:00,336 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:01,336 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:02,336 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:03,337 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:03,936 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:46:04,048 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:46:04,269 DEBUG SenderThread:2655921 [sender.py:send():379] send: files +2025-01-21 21:46:04,269 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_9b28e537c714f414ede9.mp4 with policy now +2025-01-21 21:46:04,270 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:46:04,270 DEBUG SenderThread:2655921 [sender.py:send():379] send: history +2025-01-21 21:46:04,270 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:46:04,271 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:46:04,338 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:04,503 INFO wandb-upload_1:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/i9nhftpa-media/videos/eval/0_eval_0_9b28e537c714f414ede9.mp4 +2025-01-21 21:46:04,936 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos/eval/0_eval_0_9b28e537c714f414ede9.mp4 +2025-01-21 21:46:04,937 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json +2025-01-21 21:46:04,937 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media +2025-01-21 21:46:04,937 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos +2025-01-21 21:46:04,937 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos/eval +2025-01-21 21:46:05,225 INFO wandb-upload_0:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/fxpzme77-media/videos/eval/0_eval_0_9b28e537c714f414ede9.mp4 +2025-01-21 21:46:05,337 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:05,937 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:46:06,337 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:07,338 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:08,337 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:09,015 DEBUG SystemMonitor:2655921 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:46:09,015 DEBUG SenderThread:2655921 [sender.py:send():379] send: stats +2025-01-21 21:46:09,330 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:46:09,330 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:46:09,373 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:09,473 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:46:09,938 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:46:10,338 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:11,338 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:11,939 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:46:12,338 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:13,338 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:14,338 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:14,971 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:46:15,338 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:15,940 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:46:15,940 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/events.out.tfevents.1737513909.leonmkim-ROG-Strix-G15CS-G15CS.2655690.0 +2025-01-21 21:46:16,212 INFO wandb-upload_1:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/obu5q3sh-events.out.tfevents.1737513909.leonmkim-ROG-Strix-G15CS-G15CS.2655690.0 +2025-01-21 21:46:16,338 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:17,339 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:17,941 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:46:18,339 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:19,339 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:19,941 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:46:20,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:46:20,340 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:21,340 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:22,340 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:23,340 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:23,943 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:46:24,330 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:46:24,330 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:46:24,373 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:25,340 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:25,487 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:46:26,340 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:27,341 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:28,341 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:29,341 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:30,341 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:31,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:46:31,341 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:32,341 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:33,342 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:34,342 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:35,342 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:36,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:46:36,342 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:37,343 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:38,343 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:39,016 DEBUG SenderThread:2655921 [sender.py:send():379] send: stats +2025-01-21 21:46:39,330 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:46:39,330 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:46:39,373 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:40,343 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:41,343 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:41,397 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:46:42,343 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:43,343 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:44,343 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:45,345 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:46,344 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:47,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:46:47,345 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:48,345 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:49,345 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:49,951 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:46:50,345 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:51,345 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:52,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:46:52,345 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:53,346 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:54,330 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:46:54,330 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:46:54,349 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:54,874 DEBUG SenderThread:2655921 [sender.py:send():379] send: files +2025-01-21 21:46:54,874 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ea5929cab6224759260c.mp4 with policy now +2025-01-21 21:46:54,877 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:46:54,878 DEBUG SenderThread:2655921 [sender.py:send():379] send: history +2025-01-21 21:46:54,878 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:46:54,898 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:46:54,953 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json +2025-01-21 21:46:54,953 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos/eval/1_eval_1_ea5929cab6224759260c.mp4 +2025-01-21 21:46:54,953 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos/eval +2025-01-21 21:46:55,111 INFO wandb-upload_0:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/o7cpxql2-media/videos/eval/1_eval_1_ea5929cab6224759260c.mp4 +2025-01-21 21:46:55,346 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:55,391 INFO wandb-upload_1:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/kyfwci5b-media/videos/eval/1_eval_1_ea5929cab6224759260c.mp4 +2025-01-21 21:46:55,953 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:46:56,349 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:56,954 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:46:57,352 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:57,902 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:46:58,349 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:59,349 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:46:59,955 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:47:00,350 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:01,347 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:02,348 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:02,956 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/events.out.tfevents.1737513909.leonmkim-ROG-Strix-G15CS-G15CS.2655690.0 +2025-01-21 21:47:03,348 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:03,756 DEBUG SenderThread:2655921 [sender.py:send():379] send: history +2025-01-21 21:47:03,756 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:47:03,756 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:47:03,757 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:47:03,956 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:47:03,956 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json +2025-01-21 21:47:04,348 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:05,348 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:05,957 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:47:06,349 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:07,349 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:07,958 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:47:08,349 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:08,957 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:47:09,016 DEBUG SenderThread:2655921 [sender.py:send():379] send: stats +2025-01-21 21:47:09,330 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:47:09,331 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:47:09,373 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:10,349 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:11,349 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:12,349 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:13,350 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:14,333 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:47:14,350 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:15,350 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:16,350 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:17,350 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:18,350 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:19,350 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:20,333 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:47:20,350 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:21,351 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:22,352 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:23,352 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:24,330 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:47:24,331 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:47:24,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:25,352 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:25,411 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:47:26,352 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:27,353 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:28,353 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:29,353 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:29,965 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:47:30,353 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:30,453 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:47:31,353 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:31,966 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:47:32,353 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:33,353 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:34,353 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:35,354 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:36,334 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:47:36,354 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:37,354 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:38,354 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:39,017 DEBUG SenderThread:2655921 [sender.py:send():379] send: stats +2025-01-21 21:47:39,331 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:47:39,331 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:47:39,354 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:39,968 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:47:40,354 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:41,344 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:47:41,385 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:42,368 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:42,541 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:47:42,541 DEBUG SenderThread:2655921 [sender.py:send():379] send: files +2025-01-21 21:47:42,543 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_5dc2224e3d6c0a1bac85.mp4 with policy now +2025-01-21 21:47:42,543 DEBUG SenderThread:2655921 [sender.py:send():379] send: history +2025-01-21 21:47:42,543 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:47:42,543 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:47:42,776 INFO wandb-upload_0:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/yroe57iw-media/videos/eval/2_eval_2_5dc2224e3d6c0a1bac85.mp4 +2025-01-21 21:47:42,969 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json +2025-01-21 21:47:42,969 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos/eval/2_eval_2_5dc2224e3d6c0a1bac85.mp4 +2025-01-21 21:47:42,969 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos/eval +2025-01-21 21:47:43,256 INFO wandb-upload_1:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/wg63l3vg-media/videos/eval/2_eval_2_5dc2224e3d6c0a1bac85.mp4 +2025-01-21 21:47:43,362 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:43,969 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:47:44,364 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:45,361 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:46,365 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:47,209 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:47:47,363 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:47,971 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:47:48,364 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:49,362 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:49,972 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:47:50,363 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:51,365 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:52,336 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:47:52,365 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:53,365 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:54,332 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:47:54,332 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:47:54,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:55,365 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:55,614 DEBUG SenderThread:2655921 [sender.py:send():379] send: history +2025-01-21 21:47:55,614 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:47:55,614 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:47:55,974 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/events.out.tfevents.1737513909.leonmkim-ROG-Strix-G15CS-G15CS.2655690.0 +2025-01-21 21:47:55,974 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json +2025-01-21 21:47:56,239 INFO wandb-upload_0:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/488m5on4-events.out.tfevents.1737513909.leonmkim-ROG-Strix-G15CS-G15CS.2655690.0 +2025-01-21 21:47:56,370 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:57,367 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:57,497 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:47:57,974 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:47:58,370 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:47:59,370 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:00,373 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:01,373 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:01,976 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:48:02,373 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:03,373 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:03,469 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:48:04,374 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:05,374 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:06,374 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:07,374 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:08,374 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:09,017 DEBUG SenderThread:2655921 [sender.py:send():379] send: stats +2025-01-21 21:48:09,017 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:48:09,331 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:48:09,331 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:48:09,374 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:10,375 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:11,375 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:12,375 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:13,375 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:14,337 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:48:14,375 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:15,375 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:16,375 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:17,375 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:18,375 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:19,337 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:48:19,375 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:20,376 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:21,376 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:22,376 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:23,376 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:24,331 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:48:24,331 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:48:24,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:24,396 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:48:25,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:25,984 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:48:26,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:27,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:28,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:29,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:30,337 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:48:30,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:31,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:32,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:33,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:34,377 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:35,390 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:35,567 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:48:35,987 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:48:36,287 DEBUG SenderThread:2655921 [sender.py:send():379] send: files +2025-01-21 21:48:36,287 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d7365acda7abf65fdffc.mp4 with policy now +2025-01-21 21:48:36,288 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:48:36,290 DEBUG SenderThread:2655921 [sender.py:send():379] send: history +2025-01-21 21:48:36,290 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:48:36,313 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:48:36,380 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:36,548 INFO wandb-upload_1:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/xed04jsc-media/videos/eval/3_eval_3_d7365acda7abf65fdffc.mp4 +2025-01-21 21:48:36,988 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:48:36,988 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json +2025-01-21 21:48:36,988 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos/eval/3_eval_3_d7365acda7abf65fdffc.mp4 +2025-01-21 21:48:36,988 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos/eval +2025-01-21 21:48:37,280 INFO wandb-upload_0:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/txi76w1l-media/videos/eval/3_eval_3_d7365acda7abf65fdffc.mp4 +2025-01-21 21:48:37,383 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:37,988 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:48:38,388 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:39,017 DEBUG SenderThread:2655921 [sender.py:send():379] send: stats +2025-01-21 21:48:39,334 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:48:39,335 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:48:39,388 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:40,385 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:40,965 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:48:41,389 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:41,989 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:48:42,387 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:43,390 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:43,990 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:48:44,388 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:45,391 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:46,344 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:48:46,391 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:47,392 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:48,391 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:49,398 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:50,391 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:50,678 DEBUG SenderThread:2655921 [sender.py:send():379] send: history +2025-01-21 21:48:50,678 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:48:50,679 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:48:50,992 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/events.out.tfevents.1737513909.leonmkim-ROG-Strix-G15CS-G15CS.2655690.0 +2025-01-21 21:48:50,992 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json +2025-01-21 21:48:51,393 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:51,993 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:48:52,193 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:48:52,395 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:53,395 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:54,338 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:48:54,338 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:48:54,393 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:55,393 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:55,994 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:48:56,395 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:57,394 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:57,403 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:48:58,394 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:48:59,394 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:00,394 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:01,394 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:02,394 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:03,342 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:49:03,395 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:04,395 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:05,395 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:06,396 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:07,398 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:08,342 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:49:08,398 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:09,018 DEBUG SenderThread:2655921 [sender.py:send():379] send: stats +2025-01-21 21:49:09,333 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:49:09,333 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:49:09,398 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:10,399 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:11,399 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:12,399 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:13,399 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:13,469 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:49:14,399 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:15,399 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:16,399 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:17,399 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:18,399 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:19,079 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:49:19,399 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:20,002 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:49:20,399 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:21,400 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:22,400 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:23,400 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:24,333 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:49:24,333 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:49:24,400 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:24,412 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:49:25,400 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:26,400 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:27,401 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:28,401 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:29,403 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:29,523 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:49:30,006 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:49:30,071 DEBUG SenderThread:2655921 [sender.py:send():379] send: files +2025-01-21 21:49:30,072 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_86fd9cb181be35cfea37.mp4 with policy now +2025-01-21 21:49:30,083 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:49:30,084 DEBUG SenderThread:2655921 [sender.py:send():379] send: history +2025-01-21 21:49:30,084 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:49:30,105 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:49:30,293 INFO wandb-upload_1:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/l15h5fde-media/videos/eval/4_eval_4_86fd9cb181be35cfea37.mp4 +2025-01-21 21:49:30,401 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:31,006 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:49:31,006 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json +2025-01-21 21:49:31,006 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos/eval/4_eval_4_86fd9cb181be35cfea37.mp4 +2025-01-21 21:49:31,007 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos/eval +2025-01-21 21:49:31,296 INFO wandb-upload_0:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/x92bok8q-media/videos/eval/4_eval_4_86fd9cb181be35cfea37.mp4 +2025-01-21 21:49:31,411 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:32,007 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:49:32,408 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:33,412 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:34,412 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:35,347 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:49:35,418 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:36,008 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:49:36,423 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:37,417 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:38,009 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:49:38,418 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:39,018 DEBUG SenderThread:2655921 [sender.py:send():379] send: stats +2025-01-21 21:49:39,338 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:49:39,338 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:49:39,421 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:40,009 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/events.out.tfevents.1737513909.leonmkim-ROG-Strix-G15CS-G15CS.2655690.0 +2025-01-21 21:49:40,205 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:49:40,205 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:49:40,301 INFO wandb-upload_1:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/x9ysxp1z-events.out.tfevents.1737513909.leonmkim-ROG-Strix-G15CS-G15CS.2655690.0 +2025-01-21 21:49:40,370 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:49:40,418 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:40,630 INFO wandb-upload_0:2655921 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpe9m9nj8v +2025-01-21 21:49:40,770 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:49:40,783 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:49:41,010 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/events.out.tfevents.1737513909.leonmkim-ROG-Strix-G15CS-G15CS.2655690.0 +2025-01-21 21:49:41,010 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json +2025-01-21 21:49:41,010 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 21:49:41,010 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media +2025-01-21 21:49:41,010 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/table/eval +2025-01-21 21:49:41,010 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/table +2025-01-21 21:49:41,218 INFO SenderThread:2655921 [sender.py:send_request_log_artifact():1525] logged artifact run-nf7i3zuf-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY5NTk1Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwMDU5Mg==', 'latestArtifact': None}} +2025-01-21 21:49:41,218 DEBUG SenderThread:2655921 [sender.py:send():379] send: files +2025-01-21 21:49:41,218 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-21 21:49:41,219 DEBUG SenderThread:2655921 [sender.py:send():379] send: history +2025-01-21 21:49:41,219 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:49:41,221 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:49:41,221 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:49:41,222 DEBUG SenderThread:2655921 [sender.py:send():379] send: telemetry +2025-01-21 21:49:41,222 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:49:41,222 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:49:41,222 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:49:41,223 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:49:41,223 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:49:41,223 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:49:41,223 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:49:41,223 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:49:41,224 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:49:41,224 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:49:41,224 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:49:41,224 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:49:41,225 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:49:41,225 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:49:41,225 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:49:41,225 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:49:41,225 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:49:41,226 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:49:41,226 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:49:41,226 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:49:41,226 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:49:41,227 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:49:41,227 DEBUG SenderThread:2655921 [sender.py:send():379] send: history +2025-01-21 21:49:41,227 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:49:41,227 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:49:41,227 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:49:41,405 INFO wandb-upload_1:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/6a18n8h1-media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json +2025-01-21 21:49:41,419 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:41,448 INFO wandb-upload_2:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/4ktrk0p6-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 21:49:41,691 INFO wandb-upload_2:2655921 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphngyfnzh +2025-01-21 21:49:41,721 INFO wandb-upload_0:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/1e8m1w91-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 21:49:42,010 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:49:42,010 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json +2025-01-21 21:49:42,123 INFO SenderThread:2655921 [sender.py:send_request_log_artifact():1525] logged artifact run-nf7i3zuf-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY5NTk3MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwMDU5NA==', 'latestArtifact': None}} +2025-01-21 21:49:42,123 DEBUG SenderThread:2655921 [sender.py:send():379] send: files +2025-01-21 21:49:42,123 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json with policy now +2025-01-21 21:49:42,123 DEBUG SenderThread:2655921 [sender.py:send():379] send: history +2025-01-21 21:49:42,124 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:49:42,124 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:49:42,388 INFO wandb-upload_1:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/l5ezwp1h-media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json +2025-01-21 21:49:42,419 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:43,011 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json +2025-01-21 21:49:43,419 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:44,419 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:45,419 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:46,343 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:49:46,419 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:47,421 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:48,421 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:49,420 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:50,420 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:51,421 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:52,343 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:49:52,421 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:53,421 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:54,335 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:49:54,336 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:49:54,421 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:55,421 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:56,421 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:57,416 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:49:57,422 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:58,422 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:49:59,422 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:00,422 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:01,422 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:02,422 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:02,659 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:50:02,779 DEBUG SenderThread:2655921 [sender.py:send():379] send: exit +2025-01-21 21:50:02,779 INFO SenderThread:2655921 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:50:02,779 INFO SenderThread:2655921 [sender.py:send_exit():588] handling runtime: 293 +2025-01-21 21:50:02,780 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:50:02,780 INFO SenderThread:2655921 [sender.py:send_exit():594] send defer +2025-01-21 21:50:02,780 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:50:02,780 INFO HandlerThread:2655921 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:50:02,780 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: defer +2025-01-21 21:50:02,780 INFO SenderThread:2655921 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:50:02,780 INFO SenderThread:2655921 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:50:02,780 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:50:02,781 INFO HandlerThread:2655921 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:50:02,781 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: defer +2025-01-21 21:50:02,781 INFO SenderThread:2655921 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:50:02,781 INFO SenderThread:2655921 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:50:02,781 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:50:02,781 INFO HandlerThread:2655921 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:50:02,781 INFO HandlerThread:2655921 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:50:02,781 DEBUG SystemMonitor:2655921 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:50:02,781 INFO HandlerThread:2655921 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:50:02,782 DEBUG SystemMonitor:2655921 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:50:02,782 INFO HandlerThread:2655921 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:50:02,834 INFO HandlerThread:2655921 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:50:02,834 INFO HandlerThread:2655921 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:50:02,834 INFO HandlerThread:2655921 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:50:02,835 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: defer +2025-01-21 21:50:02,835 INFO SenderThread:2655921 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:50:02,835 INFO SenderThread:2655921 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:50:02,835 DEBUG SenderThread:2655921 [sender.py:send():379] send: stats +2025-01-21 21:50:02,835 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:50:02,835 INFO HandlerThread:2655921 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:50:02,835 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: defer +2025-01-21 21:50:02,835 INFO SenderThread:2655921 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:50:02,835 INFO SenderThread:2655921 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:50:02,835 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:50:02,835 INFO HandlerThread:2655921 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:50:03,019 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/config.yaml +2025-01-21 21:50:03,019 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json +2025-01-21 21:50:04,019 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:50:09,751 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:50:09,751 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:50:09,751 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:50:09,752 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: defer +2025-01-21 21:50:09,752 INFO SenderThread:2655921 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:50:09,752 INFO SenderThread:2655921 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:50:09,753 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:50:09,753 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:50:09,753 INFO HandlerThread:2655921 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:50:09,753 DEBUG SenderThread:2655921 [sender.py:send():379] send: history +2025-01-21 21:50:09,753 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:50:09,754 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:50:09,754 DEBUG SenderThread:2655921 [sender.py:send():379] send: summary +2025-01-21 21:50:09,758 INFO SenderThread:2655921 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:50:09,758 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: defer +2025-01-21 21:50:09,758 INFO SenderThread:2655921 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:50:09,758 INFO SenderThread:2655921 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:50:09,758 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:50:09,758 INFO HandlerThread:2655921 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:50:09,759 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: defer +2025-01-21 21:50:09,759 INFO SenderThread:2655921 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:50:09,759 INFO SenderThread:2655921 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:50:09,759 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:50:09,759 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:50:09,759 INFO HandlerThread:2655921 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:50:09,759 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: defer +2025-01-21 21:50:09,759 INFO SenderThread:2655921 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:50:10,021 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json +2025-01-21 21:50:10,699 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:50:12,947 INFO SenderThread:2655921 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:50:12,947 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:50:12,948 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:50:12,948 INFO HandlerThread:2655921 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:50:12,948 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: defer +2025-01-21 21:50:12,948 INFO SenderThread:2655921 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:50:12,948 INFO SenderThread:2655921 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:50:12,951 INFO SenderThread:2655921 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:50:12,952 INFO SenderThread:2655921 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:50:12,965 INFO SenderThread:2655921 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:50:12,965 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:50:12,965 INFO HandlerThread:2655921 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:50:12,966 DEBUG SenderThread:2655921 [sender.py:send():379] send: artifact +2025-01-21 21:50:13,023 INFO Thread-12 :2655921 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:50:13,201 INFO SenderThread:2655921 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3MzQyMw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 21:50:13,201 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: defer +2025-01-21 21:50:13,201 INFO SenderThread:2655921 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:50:13,201 INFO SenderThread:2655921 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:50:13,699 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:50:14,023 INFO SenderThread:2655921 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files +2025-01-21 21:50:14,024 INFO SenderThread:2655921 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/config.yaml config.yaml +2025-01-21 21:50:14,024 INFO SenderThread:2655921 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/events.out.tfevents.1737513909.leonmkim-ROG-Strix-G15CS-G15CS.2655690.0 events.out.tfevents.1737513909.leonmkim-ROG-Strix-G15CS-G15CS.2655690.0 +2025-01-21 21:50:14,024 INFO SenderThread:2655921 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json wandb-summary.json +2025-01-21 21:50:14,024 INFO SenderThread:2655921 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/requirements.txt requirements.txt +2025-01-21 21:50:14,024 INFO SenderThread:2655921 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:50:14,024 INFO SenderThread:2655921 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log output.log +2025-01-21 21:50:14,024 INFO SenderThread:2655921 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/diff.patch diff.patch +2025-01-21 21:50:14,026 INFO SenderThread:2655921 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:50:14,026 INFO SenderThread:2655921 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos/eval/2_eval_2_5dc2224e3d6c0a1bac85.mp4 media/videos/eval/2_eval_2_5dc2224e3d6c0a1bac85.mp4 +2025-01-21 21:50:14,026 INFO SenderThread:2655921 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos/eval/1_eval_1_ea5929cab6224759260c.mp4 media/videos/eval/1_eval_1_ea5929cab6224759260c.mp4 +2025-01-21 21:50:14,027 INFO SenderThread:2655921 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos/eval/3_eval_3_d7365acda7abf65fdffc.mp4 media/videos/eval/3_eval_3_d7365acda7abf65fdffc.mp4 +2025-01-21 21:50:14,027 INFO SenderThread:2655921 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos/eval/4_eval_4_86fd9cb181be35cfea37.mp4 media/videos/eval/4_eval_4_86fd9cb181be35cfea37.mp4 +2025-01-21 21:50:14,027 INFO SenderThread:2655921 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/videos/eval/0_eval_0_9b28e537c714f414ede9.mp4 media/videos/eval/0_eval_0_9b28e537c714f414ede9.mp4 +2025-01-21 21:50:14,033 INFO SenderThread:2655921 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 21:50:14,033 INFO SenderThread:2655921 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json +2025-01-21 21:50:14,033 INFO SenderThread:2655921 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:50:14,033 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:50:14,033 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:50:14,033 INFO HandlerThread:2655921 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:50:14,033 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: defer +2025-01-21 21:50:14,033 INFO SenderThread:2655921 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:50:14,034 INFO SenderThread:2655921 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:50:14,209 INFO wandb-upload_3:2655921 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json +2025-01-21 21:50:14,251 INFO wandb-upload_0:2655921 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/config.yaml +2025-01-21 21:50:14,283 INFO wandb-upload_1:2655921 [upload_job.py:push():130] Uploaded file /tmp/tmpvak5igovwandb/zsdk0pkh-events.out.tfevents.1737513909.leonmkim-ROG-Strix-G15CS-G15CS.2655690.0 +2025-01-21 21:50:14,289 INFO wandb-upload_2:2655921 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log +2025-01-21 21:50:14,302 INFO wandb-upload_4:2655921 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/requirements.txt +2025-01-21 21:50:14,502 INFO Thread-11 (_thread_body):2655921 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:50:14,502 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:50:14,502 INFO HandlerThread:2655921 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:50:14,503 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: defer +2025-01-21 21:50:14,503 INFO SenderThread:2655921 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:50:14,503 INFO SenderThread:2655921 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:50:14,504 INFO SenderThread:2655921 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:50:14,504 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:50:14,504 INFO HandlerThread:2655921 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:50:14,504 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: defer +2025-01-21 21:50:14,504 INFO SenderThread:2655921 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:50:14,504 INFO SenderThread:2655921 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:50:14,700 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:50:14,828 INFO SenderThread:2655921 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:50:14,828 INFO SenderThread:2655921 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:50:14,828 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:50:14,828 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:50:14,828 INFO HandlerThread:2655921 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:50:14,829 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: defer +2025-01-21 21:50:14,829 INFO SenderThread:2655921 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:50:14,829 INFO SenderThread:2655921 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:50:14,829 DEBUG SenderThread:2655921 [sender.py:send():379] send: final +2025-01-21 21:50:14,829 DEBUG SenderThread:2655921 [sender.py:send():379] send: footer +2025-01-21 21:50:14,829 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:50:14,829 INFO HandlerThread:2655921 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:50:14,830 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: defer +2025-01-21 21:50:14,830 INFO SenderThread:2655921 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:50:14,831 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:50:14,831 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:50:14,831 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:50:14,831 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:50:14,832 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:50:14,832 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:50:14,833 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:14,833 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:50:14,897 DEBUG SenderThread:2655921 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:50:14,898 INFO MainThread:2655921 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:50:14,898 INFO MainThread:2655921 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:50:14,898 INFO MainThread:2655921 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:50:14,898 DEBUG HandlerThread:2655921 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:50:14,898 INFO HandlerThread:2655921 [handler.py:finish():882] shutting down handler +2025-01-21 21:50:15,833 INFO WriterThread:2655921 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/run-nf7i3zuf.wandb +2025-01-21 21:50:15,897 INFO SenderThread:2655921 [sender.py:finish():1615] shutting down sender +2025-01-21 21:50:15,898 INFO SenderThread:2655921 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:50:15,898 INFO SenderThread:2655921 [file_pusher.py:join():175] waiting for file pusher diff --git a/nf7i3zuf/wandb/debug.log b/nf7i3zuf/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e096d8266d4a9d71baac02fe65ef0ac264ed9446 --- /dev/null +++ b/nf7i3zuf/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:45:08,616 INFO MainThread:2655690 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:45:08,616 INFO MainThread:2655690 [wandb_setup.py:_flush():76] Configure stats pid to 2655690 +2025-01-21 21:45:08,616 INFO MainThread:2655690 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:45:08,616 INFO MainThread:2655690 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/settings +2025-01-21 21:45:08,616 INFO MainThread:2655690 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:45:08,616 INFO MainThread:2655690 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:45:08,616 INFO MainThread:2655690 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:45:08,616 INFO MainThread:2655690 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:45:08,617 INFO MainThread:2655690 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/logs/debug.log +2025-01-21 21:45:08,617 INFO MainThread:2655690 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/logs/debug-internal.log +2025-01-21 21:45:08,617 INFO MainThread:2655690 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:45:08,617 INFO MainThread:2655690 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_1', 'wandb_notes': '4295_1_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf'} +2025-01-21 21:45:08,617 INFO MainThread:2655690 [wandb_init.py:init():619] starting backend +2025-01-21 21:45:08,617 INFO MainThread:2655690 [wandb_init.py:init():623] setting up manager +2025-01-21 21:45:08,623 INFO MainThread:2655690 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:45:08,624 INFO MainThread:2655690 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:45:08,635 INFO MainThread:2655690 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:45:08,642 INFO MainThread:2655690 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:45:08,935 INFO MainThread:2655690 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:45:09,001 INFO MainThread:2655690 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:45:09,001 INFO MainThread:2655690 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:45:09,329 INFO MainThread:2655690 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:45:09,330 INFO MainThread:2655690 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:45:09,330 INFO MainThread:2655690 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:45:09,330 INFO MainThread:2655690 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:45:09,331 INFO MainThread:2655690 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:45:09,331 INFO MainThread:2655690 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/tb, True +2025-01-21 21:45:15,324 INFO MainThread:2655690 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:50:15,899 WARNING MsgRouterThr:2655690 [router.py:message_loop():77] message_loop has been closed diff --git a/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/config.yaml b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6bbf6e732f901bf9d095b4a6e14eaf983c9751d --- /dev/null +++ b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_1' +wandb_notes: + desc: null + value: 4295_1_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737513908 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/diff.patch b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..0d1d1c2d34392be0466354e2f5a3da5c22dfe3f1 --- /dev/null +++ b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/diff.patch @@ -0,0 +1,24 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..3a697a3 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False +@@ -144,8 +144,8 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '3465_2' # seed 2 + + # crp_D+mask+contact+act history +-wandb_run_id: '4295_0' +-# wandb_run_id: '4295_1' # seed 1 ++# wandb_run_id: '4295_0' ++wandb_run_id: '4295_1' # seed 1 + # wandb_run_id: '4295_2' # seed 2 + + # RGBD+mask+act history diff --git a/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json new file mode 100644 index 0000000000000000000000000000000000000000..5a592aac7e97094d41fce54046633e127c09be66 --- /dev/null +++ b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[10, 10, 1.0, 0.9892721175168917, 1, 0.7146403789090983, 1, 0.7492533623673705, 1, 1, 1]]} \ No newline at end of file diff --git a/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..0ec4dcf0d250240822da7b0cedb4433f337c60d9 --- /dev/null +++ b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/output.log @@ -0,0 +1,282 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737513915.325192]: resetting environment +[INFO] [1737513915.329570]: cleared current plan +[INFO] [1737513915.330572]: moving to home +[INFO] [1737513916.332677]: reached home +[INFO] [1737513916.333828]: reset action history +[INFO] [1737513919.043822]: environment reset +Starting episode 0 +[INFO] [1737513919.049217]: resetting environment +[INFO] [1737513919.055837]: cleared current plan +[INFO] [1737513919.057044]: moving to home +[INFO] [1737513920.059198]: reached home +[INFO] [1737513920.060706]: reset action history +[INFO] [1737513922.768997]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737513924.668213]: resetting environment +[INFO] [1737513924.674373]: cleared current plan +[INFO] [1737513924.676439]: moving to home +[INFO] [1737513925.679018]: reached home +[INFO] [1737513925.681273]: reset action history +[INFO] [1737513928.390132]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/episode_rosbags/episode_0_2025-01-21-21-45-28.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2655690.log +started roslaunch server http://158.130.50.37:40123/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2656068] +process[print_text-2]: started with pid [2656092] +process[rosbag_record-3]: started with pid [2656093] +started bagging! +For topic gripper_width: timestamp difference is 100008447 for nearest: 1737513930212478120 - target: 1737513930112469673 at idx 30 +For topic gripper_width: timestamp difference is 103990012 for nearest: 1737513935079170399 - target: 1737513934975180387 at idx 92 +For topic gripper_width: timestamp difference is 107503979 for nearest: 1737513939479095785 - target: 1737513939371591806 at idx 149 +For topic gripper_width: timestamp difference is -106071418 for nearest: 1737513962612457330 - target: 1737513962718528748 at idx 249 +[INFO] [1737513963.291647]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737513964.048275]: Stored episode 1. +[INFO] [1737513964.049443]: Saving video... +[INFO] [1737513964.256807]: Video saved! +Starting episode 1 +[INFO] [1737513964.271355]: resetting environment +[INFO] [1737513964.281879]: cleared current plan +[INFO] [1737513964.282116]: moving to home +[INFO] [1737513967.884663]: reached home +[INFO] [1737513967.885181]: reset action history +[INFO] [1737513970.598519]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737513974.975634]: resetting environment +[INFO] [1737513974.979834]: cleared current plan +[INFO] [1737513974.980032]: moving to home +[INFO] [1737513975.981448]: reached home +[INFO] [1737513975.981863]: reset action history +[INFO] [1737513978.690333]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/episode_rosbags/episode_1_2025-01-21-21-46-18.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2655690.log +started roslaunch server http://158.130.50.37:36365/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2656259] +process[print_text-5]: started with pid [2656260] +process[rosbag_record-6]: started with pid [2656261] +started bagging! +For topic gripper_width: timestamp difference is 113121495 for nearest: 1737513981412477479 - target: 1737513981299355984 at idx 41 +For topic gripper_width: timestamp difference is -104087193 for nearest: 1737514007512461345 - target: 1737514007616548538 at idx 249 +[INFO] [1737514013.382040]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1737514014.173567]: Stored episode 2. +[INFO] [1737514014.173856]: Saving video... +[INFO] [1737514014.850237]: Video saved! +Starting episode 2 +[INFO] [1737514014.884847]: resetting environment +[INFO] [1737514014.896215]: cleared current plan +[INFO] [1737514014.900420]: moving to home +[WARN] [1737514014.740072]: Plan exhausted +[WARN] [1737514014.792700]: Plan exhausted +[WARN] [1737514014.819269]: Plan exhausted +[WARN] [1737514014.860120]: Plan exhausted +[INFO] [1737514018.507465]: reached home +[INFO] [1737514018.509174]: reset action history +[INFO] [1737514021.236099]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737514022.822108]: resetting environment +[INFO] [1737514022.830250]: cleared current plan +[INFO] [1737514022.830425]: moving to home +[INFO] [1737514023.831876]: reached home +[INFO] [1737514023.832396]: reset action history +[INFO] [1737514026.544367]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/episode_rosbags/episode_2_2025-01-21-21-47-06.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2655690.log +started roslaunch server http://158.130.50.37:43575/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2656465] +process[print_text-8]: started with pid [2656466] +process[rosbag_record-9]: started with pid [2656467] +started bagging! +For topic gripper_width: timestamp difference is -137584051 for nearest: 1737514048412477890 - target: 1737514048550061941 at idx 249 +For topic gripper_width: timestamp difference is -103600735 for nearest: 1737514049212502247 - target: 1737514049316102982 at idx 249 +For topic gripper_width: timestamp difference is 102557403 for nearest: 1737514057179145795 - target: 1737514057076588392 at idx 249 +For topic gripper_width: timestamp difference is 104093389 for nearest: 1737514059012478904 - target: 1737514058908385515 at idx 249 +[INFO] [1737514061.343931]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1737514062.121418]: Stored episode 3. +[INFO] [1737514062.126751]: Saving video... +[INFO] [1737514062.493333]: Video saved! +Starting episode 3 +[INFO] [1737514062.534845]: resetting environment +[INFO] [1737514062.561922]: cleared current plan +[INFO] [1737514062.562187]: moving to home +[INFO] [1737514066.166375]: reached home +[INFO] [1737514066.169584]: reset action history +[INFO] [1737514068.946125]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737514075.424927]: resetting environment +[INFO] [1737514075.446240]: cleared current plan +[INFO] [1737514075.451197]: moving to home +[INFO] [1737514076.454667]: reached home +[INFO] [1737514076.458038]: reset action history +[INFO] [1737514079.178366]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/episode_rosbags/episode_3_2025-01-21-21-47-59.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2655690.log +started roslaunch server http://158.130.50.37:40345/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2656876] +process[print_text-11]: started with pid [2656877] +process[rosbag_record-12]: started with pid [2656878] +started bagging! +For topic gripper_width: timestamp difference is 112384135 for nearest: 1737514104979147012 - target: 1737514104866762877 at idx 249 +[INFO] [1737514114.820433]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[WARN] [1737514116.139291]: Plan exhausted +[WARN] [1737514116.179489]: Plan exhausted +[WARN] [1737514116.221361]: Plan exhausted +[WARN] [1737514116.262658]: Plan exhausted +[WARN] [1737514116.299374]: Plan exhausted +[INFO] [1737514115.567442]: Stored episode 4. +[INFO] [1737514115.568938]: Saving video... +[INFO] [1737514116.254436]: Video saved! +Starting episode 4 +[INFO] [1737514116.293146]: resetting environment +[INFO] [1737514116.318403]: cleared current plan +[INFO] [1737514116.318618]: moving to home +[INFO] [1737514119.921681]: reached home +[INFO] [1737514119.932554]: reset action history +[INFO] [1737514122.709720]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737514130.124718]: resetting environment +[INFO] [1737514130.144269]: cleared current plan +[INFO] [1737514130.144961]: moving to home +[INFO] [1737514131.146476]: reached home +[INFO] [1737514131.150225]: reset action history +[INFO] [1737514133.884866]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/nf7i3zuf/episode_rosbags/episode_4_2025-01-21-21-48-53.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2655690.log +started roslaunch server http://158.130.50.37:41383/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2657080] +process[print_text-14]: started with pid [2657081] +process[rosbag_record-15]: started with pid [2657082] +started bagging! +For topic gripper_width: timestamp difference is 120151827 for nearest: 1737514159079119513 - target: 1737514158958967686 at idx 249 +[INFO] [1737514168.715479]: Storing episode... +[print_text-14] killing on exit +[rosbag_record-15] killing on exit[pub_text-13] killing on exit +[WARN] [1737514170.028251]: Plan exhausted +[WARN] [1737514170.071960]: Plan exhausted +[WARN] [1737514170.102356]: Plan exhausted +[INFO] [1737514169.520819]: Stored episode 5. +[INFO] [1737514169.525722]: Saving video... +[INFO] [1737514169.988703]: Video saved! +[INFO] [1737514170.094989]: resetting environment +[INFO] [1737514170.120949]: cleared current plan +[INFO] [1737514170.122058]: moving to home +[INFO] [1737514173.728041]: reached home +[INFO] [1737514173.732797]: reset action history +[INFO] [1737514176.577564]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/requirements.txt b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-metadata.json b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..bc012884c4280673b04bc5bdf2b515fb24db87b4 --- /dev/null +++ b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T02:45:09.022974", + "startedAt": "2025-01-22T02:45:08.614701", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3742.508625, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4491.531, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.014, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 406.1821403503418 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8330ce4db53de4f27b906ea09ea914666ac8166e --- /dev/null +++ b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "9b28e537c714f414ede9ebbbc7da92ec34cde9fa3391484321848b114cabc505", "size": 1475941, "path": "media/videos/eval/0_eval_0_9b28e537c714f414ede9.mp4"}, "global_step": 665, "_timestamp": 1737514180.6418924, "_runtime": 272.0176453590393, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "ea5929cab6224759260c660ed866bdf16ef354d196faba0bd4d3062847381cfe", "size": 1478121, "path": "media/videos/eval/1_eval_1_ea5929cab6224759260c.mp4"}, "eval/num_success": 5.0, "episode": 10.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "5dc2224e3d6c0a1bac85ed67f34d58d253381326fd534bc0f4a132735437246d", "size": 1473697, "path": "media/videos/eval/2_eval_2_5dc2224e3d6c0a1bac85.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "d7365acda7abf65fdffc37e78f5f71610b644f7286226caf3dfadf4c836d4798", "size": 1505668, "path": "media/videos/eval/3_eval_3_d7365acda7abf65fdffc.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "86fd9cb181be35cfea37fd46ca43b852bedc04d329d5a1d062319f2c3242411a", "size": 1480928, "path": "media/videos/eval/4_eval_4_86fd9cb181be35cfea37.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://a36387o2pwqrw8mptn0anqibi0mf8fxksiegn61v6x7d5ourwvr5rpoo9z7sx13kn4d52i0qv292d4m832u58jpsr87f4gny1x32y10mv59t4bq70i76su2qnaw6nphf/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://2597serfdkxfec6fo2sn8eb75ludl0nnx0lvfwrsky1a8wmn2bvdeyv25h0wpl0u50ozhs72kudafanim8l4sa0wtwna0e485wy7yl5qhnjr8fa6bwr6au3g1f63yack:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 10, "total_num_episodes": 10, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9892721175168917, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.7146403789090983, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.7492533623673705, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "802de47402cd97ff16985e7f9caeff0984676b3a683258550eeb2d6f43f4d0e7", "size": 420, "artifact_path": "wandb-client-artifact://o9svyvk4lyq5up1yvl2f890qz0xfm7y1od4jt3wb32hc5labnhaxn07fjgfjqu8xvgnimnxqjeswfxc4qpczjpevtci1ays565pxb4xsj7vc9ptcksauncdx5kq7unl7/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://jnoe63wwwegepe517t3hbh80qgo1korfad8xh7rj4i2x105w7a2iq4z2k1xnk59xjp0hhk14cxpxdpekhfgdtkfe1cuy0b4m7a1s5qh5dn3l6fou4quzy4vhakvk3v8e:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 293}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9892721176147461, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.7146403789520264, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.7492533326148987, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/run-nf7i3zuf.wandb b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/run-nf7i3zuf.wandb new file mode 100644 index 0000000000000000000000000000000000000000..3e8dc718c459f6f91d84656d288899e8093fbe34 Binary files /dev/null and b/nf7i3zuf/wandb/run-20250121_214508-nf7i3zuf/run-nf7i3zuf.wandb differ diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf16b257c3ff326b02f5405012760e806eacb9be --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4312578d88d22b131a6d54f17dc3a85a1cbd2baad9212b44072bd2f4867d0b9 +size 910332934 diff --git a/snapshot_11249.pt b/snapshot_11249.pt new file mode 100644 index 0000000000000000000000000000000000000000..16940bd65afd033d32dbf9fbd3c4976c93beda9b --- /dev/null +++ b/snapshot_11249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f6a8112086cd18c95bcd923578de368e5e86bbd9f4d568eb58ee00e5a95a151 +size 910332934 diff --git a/snapshot_11499.pt b/snapshot_11499.pt new file mode 100644 index 0000000000000000000000000000000000000000..5898e3aa5b6a533bf536b79fa2df5482b1af01e5 --- /dev/null +++ b/snapshot_11499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:162a726a24c220c9897b29b2492e8589078cbb9b4583d50e0d1d78177a3691e8 +size 910332934 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..32f55adc03605cd9891eb6c0f579bb1cf3684951 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe60eb8309b3842381a2aa86c9380337206a35a4c7a22bef930a87d2c308c6e5 +size 910332934 diff --git a/snapshot_12749.pt b/snapshot_12749.pt new file mode 100644 index 0000000000000000000000000000000000000000..c60e2c2993035f9c71aa6192c290d3bd029a59e9 --- /dev/null +++ b/snapshot_12749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ffa90c0df16a3f5d04465466e05b7f1342a1b26c8ec6709a49bcab340b0974c +size 910332934 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..72de6c4acc2b0973d765bc881bd7ea05d912b2b2 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e557481ba855ec4a9153bafe45436eeca3e5311bca58e671ba45206d7b05ba01 +size 910332934 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..b552aa4cd908853943ff59a38c10051a102f2251 --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7e9096025e56f62b2e081e9e63e47d814498bb867e8641aeeab5af1d1837acd +size 910332934 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..04c5d5c0accd6c695bc1f5c1521674c7717b6ef4 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dac53119176670128ab9b1aab9b7c543620391793c6701c893dc7f62d9082587 +size 910332424 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..5abaa4ffd8e2fec9ccacd2597875fb817030182a --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20b3b8b545e04703210d53c788a6585645ace8cb113d09165ff988a2ac42e72c +size 910332424 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..dabc335ac7a0489d4bf0902a07e07e03a53ae179 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b803aed171a6658361132853cf19bc3519f60cf89784be2297317d5c3042d12 +size 910332424 diff --git a/snapshot_7999.pt b/snapshot_7999.pt new file mode 100644 index 0000000000000000000000000000000000000000..800859e26ce01bdd76ce942316d7c0227b37c8f0 --- /dev/null +++ b/snapshot_7999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dafdbf5219cb63e4f14fe41e4e31bb7c3584bff694153a29ff7c497d710048b6 +size 910332424 diff --git a/snapshot_8999.pt b/snapshot_8999.pt new file mode 100644 index 0000000000000000000000000000000000000000..6cbc5bdc5c5dfe989970fa08583e446557a8db0f --- /dev/null +++ b/snapshot_8999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7489990dd7c65b2180f48c25a705cadb18c3d8dad8ca87ad5238617dccc1156b +size 910332934 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..f20b5888395c58e3c952ae96db508b84dea89ab8 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58a92bbbb2b973eb3a5bdb9e995d5c47c0bb4a1b4f908da57dd340c1552764a4 +size 910332934 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..b552aa4cd908853943ff59a38c10051a102f2251 --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7e9096025e56f62b2e081e9e63e47d814498bb867e8641aeeab5af1d1837acd +size 910332934 diff --git a/tqpjw9g6/.hydra/config.yaml b/tqpjw9g6/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2154a6acb64e2809a71d0adfc508bd71e0da7dde --- /dev/null +++ b/tqpjw9g6/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/tqpjw9g6/.hydra/hydra.yaml b/tqpjw9g6/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8605f68dcca9b173c5b2f33ed9e96c30fd33fda --- /dev/null +++ b/tqpjw9g6/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/tqpjw9g6 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/tqpjw9g6/.hydra/overrides.yaml b/tqpjw9g6/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/tqpjw9g6/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/tqpjw9g6/eval_policy.log b/tqpjw9g6/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..21789a3f4adf0311622ffc49222109252131172c --- /dev/null +++ b/tqpjw9g6/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-21 21:44:58,881][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:44:58,884][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-21 21:45:02,958][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/zxwyz3xz/.hydra/config.yaml b/zxwyz3xz/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2154a6acb64e2809a71d0adfc508bd71e0da7dde --- /dev/null +++ b/zxwyz3xz/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/zxwyz3xz/.hydra/hydra.yaml b/zxwyz3xz/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32cacc05337f11d1603768c152495f9c779a805b --- /dev/null +++ b/zxwyz3xz/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/zxwyz3xz/.hydra/overrides.yaml b/zxwyz3xz/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/zxwyz3xz/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/zxwyz3xz/episode_rosbags/aligned_depth_to_color_K.npy b/zxwyz3xz/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/zxwyz3xz/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/zxwyz3xz/episode_rosbags/cam_tf_world.npy b/zxwyz3xz/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/zxwyz3xz/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/zxwyz3xz/episode_rosbags/color_K.npy b/zxwyz3xz/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/zxwyz3xz/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/zxwyz3xz/episode_rosbags/depth_K.npy b/zxwyz3xz/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/zxwyz3xz/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/zxwyz3xz/eval_robot.log b/zxwyz3xz/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..0db82241fd33ee297b34dd5cf50c6093ed8e5ddd --- /dev/null +++ b/zxwyz3xz/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 22:16:34,341][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 22:16:34,344][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/zxwyz3xz/eval_video/0_eval.mp4 b/zxwyz3xz/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4d54d67fe89e5d748be742eede66b2a584557abf --- /dev/null +++ b/zxwyz3xz/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54e7fe314d9c986a5f0c339b114e70967262f53507592df0bdb70c184643fc31 +size 1473054 diff --git a/zxwyz3xz/eval_video/1_eval.mp4 b/zxwyz3xz/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e3bbac8eaff7ab815f69b4b2d5c021614b954aad --- /dev/null +++ b/zxwyz3xz/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eac294d6ae1c852d4b2d5e31891d40e9a13ff5209555f8040faf7101096d7db9 +size 1479950 diff --git a/zxwyz3xz/eval_video/2_eval.mp4 b/zxwyz3xz/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9f8c1441af0b2f25db351e02210481bdc3544f7 --- /dev/null +++ b/zxwyz3xz/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1e3df16fecb0ef689e4dde6e759b4639cd5eb141e92f708d145e9427a95270e +size 1471318 diff --git a/zxwyz3xz/eval_video/3_eval.mp4 b/zxwyz3xz/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64511be74df573103663ea7db4d7cc6efbea28a8 --- /dev/null +++ b/zxwyz3xz/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8f87ed816b532325ab6470a29da11664f2bdb55b46547c19c28ce3879ae3f16 +size 1518514 diff --git a/zxwyz3xz/eval_video/4_eval.mp4 b/zxwyz3xz/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..daaf7639aeb8dc15d5fc2ed3735c60e15590ed9f --- /dev/null +++ b/zxwyz3xz/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63c7fb6457e58c00fb2ccb5dff55bc85027fcb297dd43bbc0c0dcca2ba00d203 +size 1500970 diff --git a/zxwyz3xz/tb/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 b/zxwyz3xz/tb/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 new file mode 100644 index 0000000000000000000000000000000000000000..c37782ae91faa33fcf40911fd627adf3673029de --- /dev/null +++ b/zxwyz3xz/tb/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:706a660186cb89766cb5da9ae85aaff34f5aabfa0454e09bd1580295654d00b8 +size 6539 diff --git a/zxwyz3xz/wandb/debug-internal.log b/zxwyz3xz/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a2f7df02fa6dabc0ac08ef292298ffc4b6cd6fa6 --- /dev/null +++ b/zxwyz3xz/wandb/debug-internal.log @@ -0,0 +1,826 @@ +2025-01-21 22:16:40,313 INFO StreamThr :2664121 [internal.py:wandb_internal():85] W&B internal server running at pid: 2664121, started at: 2025-01-21 22:16:40.312824 +2025-01-21 22:16:40,314 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status +2025-01-21 22:16:40,324 INFO WriterThread:2664121 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/run-zxwyz3xz.wandb +2025-01-21 22:16:40,325 DEBUG SenderThread:2664121 [sender.py:send():379] send: header +2025-01-21 22:16:40,336 DEBUG SenderThread:2664121 [sender.py:send():379] send: run +2025-01-21 22:16:40,594 INFO SenderThread:2664121 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files +2025-01-21 22:16:40,594 INFO SenderThread:2664121 [sender.py:_start_run_threads():1188] run started: zxwyz3xz with start time 1737515800.313398 +2025-01-21 22:16:40,608 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 22:16:40,608 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: check_version +2025-01-21 22:16:40,682 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 22:16:40,686 DEBUG HandlerThread:2664121 [system_info.py:__init__():26] System info init +2025-01-21 22:16:40,686 DEBUG HandlerThread:2664121 [system_info.py:__init__():41] System info init done +2025-01-21 22:16:40,687 INFO HandlerThread:2664121 [system_monitor.py:start():194] Starting system monitor +2025-01-21 22:16:40,687 INFO SystemMonitor:2664121 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 22:16:40,687 INFO HandlerThread:2664121 [system_monitor.py:probe():214] Collecting system info +2025-01-21 22:16:40,687 INFO SystemMonitor:2664121 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 22:16:40,688 INFO SystemMonitor:2664121 [interfaces.py:start():188] Started disk monitoring +2025-01-21 22:16:40,689 INFO SystemMonitor:2664121 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 22:16:40,690 INFO SystemMonitor:2664121 [interfaces.py:start():188] Started memory monitoring +2025-01-21 22:16:40,691 INFO SystemMonitor:2664121 [interfaces.py:start():188] Started network monitoring +2025-01-21 22:16:40,701 DEBUG HandlerThread:2664121 [system_info.py:probe():152] Probing system +2025-01-21 22:16:40,703 DEBUG HandlerThread:2664121 [system_info.py:_probe_git():137] Probing git +2025-01-21 22:16:40,708 DEBUG HandlerThread:2664121 [system_info.py:_probe_git():145] Probing git done +2025-01-21 22:16:40,708 DEBUG HandlerThread:2664121 [system_info.py:probe():200] Probing system done +2025-01-21 22:16:40,708 DEBUG HandlerThread:2664121 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T03:16:40.702062', 'startedAt': '2025-01-22T03:16:40.305946', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3565.2328749999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4502.166, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 465.6271781921387}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 22:16:40,708 INFO HandlerThread:2664121 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 22:16:40,708 INFO HandlerThread:2664121 [system_monitor.py:probe():227] Publishing system info +2025-01-21 22:16:40,708 DEBUG HandlerThread:2664121 [system_info.py:_save_code():44] Saving code +2025-01-21 22:16:40,710 DEBUG HandlerThread:2664121 [system_info.py:_save_code():67] Saving code done +2025-01-21 22:16:40,710 DEBUG HandlerThread:2664121 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 22:16:40,772 DEBUG HandlerThread:2664121 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 22:16:40,772 INFO HandlerThread:2664121 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 22:16:40,776 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:16:40,776 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 22:16:40,776 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 22:16:40,776 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 22:16:41,012 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 22:16:41,012 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: python_packages +2025-01-21 22:16:41,012 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:16:41,012 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:16:41,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:41,015 INFO HandlerThread:2664121 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/tb" + save: true +} +control { + relay_id: "bb8cd6a3072c49c789c67331dd91f729" +} +_info { + stream_id: "zxwyz3xz" +} + +2025-01-21 22:16:41,015 DEBUG HandlerThread:2664121 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 22:16:41,049 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/4m36kohu-wandb-metadata.json +2025-01-21 22:16:41,050 INFO wandb-upload_1:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/p0gake1q-code/FISH/eval_robot.py +2025-01-21 22:16:41,066 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/6zjdpwkl-diff.patch +2025-01-21 22:16:41,110 DEBUG SenderThread:2664121 [sender.py:send():379] send: telemetry +2025-01-21 22:16:41,111 DEBUG SenderThread:2664121 [sender.py:send():379] send: tbrecord +2025-01-21 22:16:41,111 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:16:41,111 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 with policy live +2025-01-21 22:16:41,304 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/10cc6yr0-events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:16:41,595 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/diff.patch +2025-01-21 22:16:41,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-metadata.json +2025-01-21 22:16:41,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/code/FISH/eval_robot.py +2025-01-21 22:16:41,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/requirements.txt +2025-01-21 22:16:41,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:16:41,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:16:41,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/code +2025-01-21 22:16:41,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/code/FISH +2025-01-21 22:16:42,012 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:43,012 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:43,595 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:16:44,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:44,563 DEBUG SenderThread:2664121 [sender.py:send():379] send: config +2025-01-21 22:16:45,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:45,576 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:16:45,595 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:16:46,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:47,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:47,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:16:48,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:49,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:49,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:16:50,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:51,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:16:51,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:51,597 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:16:52,053 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:53,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:53,598 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:16:54,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:55,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:56,012 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:16:56,012 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:16:56,053 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:56,113 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:16:57,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:58,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:59,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:00,015 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:01,015 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:01,600 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:02,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:02,015 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:03,015 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:04,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:05,025 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:05,601 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:06,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:07,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:07,025 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:08,025 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:09,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:09,603 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:10,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:11,012 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:17:11,012 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:17:11,053 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:12,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:12,174 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:12,604 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/config.yaml +2025-01-21 22:17:13,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:14,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:15,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:16,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:17,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:18,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:18,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:19,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:20,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:21,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:22,028 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:23,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:23,028 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:24,028 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:25,028 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:26,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:17:26,013 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:17:26,057 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:27,028 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:28,028 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:28,184 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:29,029 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:30,029 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:31,029 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:32,029 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:33,029 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:34,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:34,030 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:35,030 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:36,030 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:37,030 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:38,053 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:38,973 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:17:38,973 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 with policy now +2025-01-21 22:17:38,974 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:17:38,974 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:17:38,975 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:17:38,976 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:17:39,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:39,258 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/iu2ellmg-media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 +2025-01-21 22:17:39,611 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:39,611 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:17:39,611 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 +2025-01-21 22:17:39,612 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval +2025-01-21 22:17:39,612 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos +2025-01-21 22:17:39,612 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media +2025-01-21 22:17:39,944 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/t8bfbyra-media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 +2025-01-21 22:17:39,989 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:40,031 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:40,691 DEBUG SystemMonitor:2664121 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 22:17:40,692 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:17:41,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:17:41,013 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:17:41,053 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:41,612 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:42,031 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:43,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:43,612 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:44,031 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:45,031 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:45,304 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:46,031 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:47,031 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:47,614 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:48,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:48,614 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:17:48,865 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/8lb3sby5-events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:17:49,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:49,614 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:50,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:50,621 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:51,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:51,615 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:52,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:53,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:53,616 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:54,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:55,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:56,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:17:56,013 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:17:56,057 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:56,082 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:57,033 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:58,033 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:59,033 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:00,033 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:01,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:02,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:02,033 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:03,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:04,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:05,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:06,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:07,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:07,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:08,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:09,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:10,035 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:10,700 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:18:11,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:18:11,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:18:11,057 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:12,035 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:12,170 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:13,035 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:14,035 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:15,035 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:16,035 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:17,036 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:18,015 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:18,036 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:19,036 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:20,038 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:21,037 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:22,037 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:23,015 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:23,037 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:24,037 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:25,037 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:26,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:18:26,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:18:26,061 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:27,038 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:27,627 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:28,039 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:28,054 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:28,101 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:18:28,101 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:18:28,102 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 with policy now +2025-01-21 22:18:28,102 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:18:28,102 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:18:28,124 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:18:28,366 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/tcov7fon-media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 +2025-01-21 22:18:28,627 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:18:28,627 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 +2025-01-21 22:18:28,628 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval +2025-01-21 22:18:28,943 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/ddsqehya-media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 +2025-01-21 22:18:29,046 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:29,628 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:30,045 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:31,060 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:32,040 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:33,043 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:33,630 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:33,773 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:34,054 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:35,042 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:35,631 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:36,043 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:37,043 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:38,043 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:39,017 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:39,045 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:40,043 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:40,693 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:18:41,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:18:41,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:18:41,057 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:42,044 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:43,044 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:44,045 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:44,091 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:45,046 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:46,044 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:46,435 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:18:46,435 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:18:46,436 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:18:46,635 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:18:46,635 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:18:47,044 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:47,635 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:48,044 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:49,044 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:49,117 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:49,637 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:50,144 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:51,123 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:51,637 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:52,124 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:53,124 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:54,124 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:54,625 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:55,124 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:55,639 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:56,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:18:56,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:18:56,124 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:57,125 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:58,125 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:59,125 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:00,125 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:00,130 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:01,125 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:02,125 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:03,125 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:04,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:05,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:06,020 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:06,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:07,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:08,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:09,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:10,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:10,693 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:19:11,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:19:11,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:19:11,105 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:11,128 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:12,127 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:13,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:14,127 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:15,127 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:16,127 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:17,020 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:17,127 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:18,127 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:19,127 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:20,128 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:21,128 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:22,022 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:22,128 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:23,128 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:24,128 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:25,128 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:25,650 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:25,925 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:19:25,925 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 with policy now +2025-01-21 22:19:25,925 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:19:25,926 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:19:25,926 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:19:25,941 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:19:26,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:19:26,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:19:26,131 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:26,176 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/9x8g5qjh-media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 +2025-01-21 22:19:26,650 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:19:26,651 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 +2025-01-21 22:19:26,651 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval +2025-01-21 22:19:26,957 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/w8aq67zh-media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 +2025-01-21 22:19:27,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:27,154 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:27,651 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:28,137 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:29,143 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:30,145 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:31,146 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:31,652 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:32,146 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:32,347 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:33,144 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:33,653 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:34,142 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:34,653 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:19:34,896 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/lrx0e4nt-events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:19:35,143 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:35,276 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:19:35,277 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:19:35,277 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:19:35,653 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:19:35,654 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:36,143 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:37,143 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:37,654 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:38,024 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:38,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:39,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:39,655 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:40,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:40,693 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:19:41,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:19:41,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:19:41,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:41,656 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:42,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:43,145 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:43,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:44,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:45,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:46,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:47,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:48,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:49,022 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:49,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:50,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:51,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:52,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:53,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:54,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:55,022 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:55,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:56,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:19:56,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:19:56,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:57,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:58,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:59,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:59,662 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:00,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:00,841 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:01,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:02,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:03,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:04,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:05,150 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:05,664 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:06,022 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:06,150 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:07,150 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:08,150 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:09,150 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:10,151 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:10,694 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:20:11,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:20:11,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:20:11,120 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:11,152 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:12,152 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:13,152 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:14,152 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:15,156 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:15,189 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:20:15,189 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 with policy now +2025-01-21 22:20:15,229 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:20:15,229 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:20:15,230 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:20:15,234 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:20:15,447 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/9f5ij4ht-media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 +2025-01-21 22:20:15,667 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:20:15,667 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:15,667 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 +2025-01-21 22:20:15,667 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval +2025-01-21 22:20:15,959 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/9qsuslpj-media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 +2025-01-21 22:20:16,156 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:17,025 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:17,163 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:17,668 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:18,177 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:19,164 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:19,668 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:20,164 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:21,172 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:21,669 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:22,158 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:22,713 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:23,168 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:23,670 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:24,168 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:25,161 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:26,019 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:20:26,020 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:20:26,162 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:27,166 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:28,104 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:28,164 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:29,161 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:29,344 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:20:29,344 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:20:29,345 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:20:29,672 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:20:29,672 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:20:30,169 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:31,170 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:31,673 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:32,172 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:33,172 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:33,195 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:34,170 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:35,170 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:35,674 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:36,170 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:37,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:38,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:39,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:39,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:40,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:40,694 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:20:41,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:20:41,015 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:20:41,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:42,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:43,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:44,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:44,184 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:45,172 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:46,172 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:47,172 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:48,175 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:49,175 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:49,679 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:49,725 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:50,175 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:51,175 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:52,175 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:53,175 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:54,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:55,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:55,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:56,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:20:56,015 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:20:56,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:57,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:57,681 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:58,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:59,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:00,137 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:00,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:01,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:02,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:03,177 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:04,177 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:05,177 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:06,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:06,177 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:07,177 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:08,180 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:09,193 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:09,542 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:21:09,542 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 with policy now +2025-01-21 22:21:09,545 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:21:09,545 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:21:09,545 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:09,581 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:09,756 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:21:09,767 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:09,767 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 +2025-01-21 22:21:09,767 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval +2025-01-21 22:21:09,794 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/bz2lyf2v-media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 +2025-01-21 22:21:10,080 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/d87scncr-media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 +2025-01-21 22:21:10,215 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:10,695 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:21:11,029 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:21:11,029 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:21:11,139 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:11,193 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:11,756 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:12,199 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:13,221 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:14,200 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:15,199 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:15,758 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:16,204 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:17,093 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:17,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:17,758 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:18,203 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:19,202 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:20,214 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:21,213 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:22,211 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:23,041 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:23,209 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:24,216 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:24,761 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:21:25,013 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/f59lbj9f-events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:21:25,223 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:25,412 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:21:25,413 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:25,413 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:25,433 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:21:25,433 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:21:25,668 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:21:25,761 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:21:25,761 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:25,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:21:25,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media +2025-01-21 22:21:25,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table +2025-01-21 22:21:25,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval +2025-01-21 22:21:25,952 INFO wandb-upload_0:2664121 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp35brkh83 +2025-01-21 22:21:25,956 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/vob1phb9-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:21:26,017 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:21:26,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:21:26,040 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:21:26,208 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:26,491 INFO SenderThread:2664121 [sender.py:send_request_log_artifact():1525] logged artifact run-zxwyz3xz-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzczMzAwNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwNzM1Ng==', 'latestArtifact': None}} +2025-01-21 22:21:26,492 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:21:26,492 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-21 22:21:26,492 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:21:26,492 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,492 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,493 DEBUG SenderThread:2664121 [sender.py:send():379] send: telemetry +2025-01-21 22:21:26,493 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,496 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,497 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,497 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,497 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,497 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,498 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,498 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,498 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,498 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,499 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,499 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,499 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,499 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,500 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,500 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,500 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,500 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,501 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,501 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,501 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,501 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,501 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:21:26,650 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:21:26,743 INFO wandb-upload_1:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/w97akgo4-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:21:26,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:21:26,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:21:26,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-21 22:21:26,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval +2025-01-21 22:21:26,992 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/5p5mfol4-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-21 22:21:27,009 INFO wandb-upload_1:2664121 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpj108ekry +2025-01-21 22:21:27,208 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:27,402 INFO SenderThread:2664121 [sender.py:send_request_log_artifact():1525] logged artifact run-zxwyz3xz-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzczMzAyOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwNzM1OQ==', 'latestArtifact': None}} +2025-01-21 22:21:27,402 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:21:27,403 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json with policy now +2025-01-21 22:21:27,403 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:21:27,403 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:27,403 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:27,696 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/nsx7eczf-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-21 22:21:27,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:21:27,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:28,208 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:28,404 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:29,208 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:30,208 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:31,208 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:32,208 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:33,209 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:34,035 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:34,209 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:34,765 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/config.yaml +2025-01-21 22:21:35,209 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:36,209 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:37,209 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:38,209 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:39,214 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:40,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:40,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:40,695 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:21:41,017 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:21:41,017 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:21:41,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:42,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:43,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:44,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:45,114 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:45,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:46,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:46,725 DEBUG SenderThread:2664121 [sender.py:send():379] send: exit +2025-01-21 22:21:46,725 INFO SenderThread:2664121 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 22:21:46,725 INFO SenderThread:2664121 [sender.py:send_exit():588] handling runtime: 306 +2025-01-21 22:21:46,726 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:46,726 INFO SenderThread:2664121 [sender.py:send_exit():594] send defer +2025-01-21 22:21:46,726 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:46,726 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 22:21:46,726 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:46,726 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 22:21:46,726 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 1 +2025-01-21 22:21:46,726 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:46,726 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 22:21:46,727 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:46,727 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 22:21:46,727 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 2 +2025-01-21 22:21:46,727 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:46,727 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 22:21:46,727 INFO HandlerThread:2664121 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 22:21:46,727 DEBUG SystemMonitor:2664121 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 22:21:46,727 INFO HandlerThread:2664121 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 22:21:46,727 DEBUG SystemMonitor:2664121 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 22:21:46,727 INFO HandlerThread:2664121 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 22:21:46,770 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:21:46,781 INFO HandlerThread:2664121 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 22:21:46,781 INFO HandlerThread:2664121 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 22:21:46,782 INFO HandlerThread:2664121 [interfaces.py:finish():200] Joined network monitor +2025-01-21 22:21:46,782 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:46,782 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 22:21:46,782 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 3 +2025-01-21 22:21:46,782 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:21:46,782 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:46,782 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 22:21:46,783 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:46,783 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 22:21:46,783 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 4 +2025-01-21 22:21:46,783 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:46,783 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 22:21:47,771 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:53,418 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:21:53,418 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:53,418 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 22:21:53,419 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:53,419 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 22:21:53,419 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 5 +2025-01-21 22:21:53,420 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:21:53,420 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:21:53,420 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:53,420 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:53,420 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 22:21:53,421 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:53,421 DEBUG SenderThread:2664121 [sender.py:send():379] send: summary +2025-01-21 22:21:53,421 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:53,422 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:53,422 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 22:21:53,422 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 6 +2025-01-21 22:21:53,422 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:53,422 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 22:21:53,422 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:53,422 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 22:21:53,422 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 7 +2025-01-21 22:21:53,422 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:53,422 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:53,422 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 22:21:53,422 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:53,423 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 22:21:53,727 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:21:53,773 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:21:55,331 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 8 +2025-01-21 22:21:55,331 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:21:55,332 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:55,332 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 22:21:55,332 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:55,332 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 22:21:55,332 INFO SenderThread:2664121 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 22:21:55,335 INFO SenderThread:2664121 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 22:21:55,337 INFO SenderThread:2664121 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 22:21:55,354 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 9 +2025-01-21 22:21:55,354 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:55,354 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 22:21:55,354 DEBUG SenderThread:2664121 [sender.py:send():379] send: artifact +2025-01-21 22:21:55,578 INFO SenderThread:2664121 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3MzQyMw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 22:21:55,578 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:55,578 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 22:21:55,578 INFO SenderThread:2664121 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 22:21:55,727 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:21:55,774 INFO SenderThread:2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:55,774 INFO SenderThread:2664121 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/config.yaml config.yaml +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json wandb-summary.json +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/requirements.txt requirements.txt +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-metadata.json wandb-metadata.json +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log output.log +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/diff.patch diff.patch +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 22:21:55,777 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 +2025-01-21 22:21:55,780 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 +2025-01-21 22:21:55,780 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 +2025-01-21 22:21:55,780 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 +2025-01-21 22:21:55,781 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 +2025-01-21 22:21:55,783 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-21 22:21:55,784 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:21:55,784 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 10 +2025-01-21 22:21:55,784 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:21:55,784 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:55,784 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 22:21:55,784 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:55,784 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 22:21:55,784 INFO SenderThread:2664121 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:21:55,971 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:21:56,004 INFO wandb-upload_4:2664121 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:56,014 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/config.yaml +2025-01-21 22:21:56,016 INFO wandb-upload_1:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/mat6ces8-events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:21:56,028 INFO wandb-upload_3:2664121 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/requirements.txt +2025-01-21 22:21:56,228 INFO Thread-11 (_thread_body):2664121 [sender.py:transition_state():614] send defer: 11 +2025-01-21 22:21:56,228 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:56,228 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 22:21:56,229 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:56,229 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 22:21:56,229 INFO SenderThread:2664121 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 22:21:56,230 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 12 +2025-01-21 22:21:56,230 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:56,230 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 22:21:56,230 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:56,230 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 22:21:56,230 INFO SenderThread:2664121 [file_stream.py:finish():601] file stream finish called +2025-01-21 22:21:56,445 INFO SenderThread:2664121 [file_stream.py:finish():605] file stream finish is done +2025-01-21 22:21:56,445 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 13 +2025-01-21 22:21:56,445 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:56,445 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 22:21:56,446 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:56,446 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 22:21:56,446 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 14 +2025-01-21 22:21:56,446 DEBUG SenderThread:2664121 [sender.py:send():379] send: final +2025-01-21 22:21:56,446 DEBUG SenderThread:2664121 [sender.py:send():379] send: footer +2025-01-21 22:21:56,446 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:56,446 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 22:21:56,446 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:56,446 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 22:21:56,447 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:21:56,447 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 22:21:56,447 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:21:56,448 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: server_info +2025-01-21 22:21:56,448 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:21:56,449 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 22:21:56,449 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 22:21:56,450 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:56,513 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:21:56,513 INFO MainThread:2664121 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 22:21:56,514 INFO MainThread:2664121 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 22:21:56,514 INFO MainThread:2664121 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 22:21:56,514 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 22:21:56,514 INFO HandlerThread:2664121 [handler.py:finish():882] shutting down handler +2025-01-21 22:21:57,450 INFO WriterThread:2664121 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/run-zxwyz3xz.wandb +2025-01-21 22:21:57,513 INFO SenderThread:2664121 [sender.py:finish():1615] shutting down sender +2025-01-21 22:21:57,513 INFO SenderThread:2664121 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:21:57,514 INFO SenderThread:2664121 [file_pusher.py:join():175] waiting for file pusher diff --git a/zxwyz3xz/wandb/debug.log b/zxwyz3xz/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..03c270d4116113d9fcd0cbc9b2ac0832c34e6700 --- /dev/null +++ b/zxwyz3xz/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Configure stats pid to 2663926 +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/settings +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 22:16:40,308 INFO MainThread:2663926 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/logs/debug.log +2025-01-21 22:16:40,308 INFO MainThread:2663926 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/logs/debug-internal.log +2025-01-21 22:16:40,308 INFO MainThread:2663926 [wandb_init.py:init():569] calling init triggers +2025-01-21 22:16:40,308 INFO MainThread:2663926 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_1', 'wandb_notes': '4295_1_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz'} +2025-01-21 22:16:40,308 INFO MainThread:2663926 [wandb_init.py:init():619] starting backend +2025-01-21 22:16:40,308 INFO MainThread:2663926 [wandb_init.py:init():623] setting up manager +2025-01-21 22:16:40,312 INFO MainThread:2663926 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 22:16:40,313 INFO MainThread:2663926 [wandb_init.py:init():631] backend started and connected +2025-01-21 22:16:40,324 INFO MainThread:2663926 [wandb_init.py:init():720] updated telemetry +2025-01-21 22:16:40,333 INFO MainThread:2663926 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 22:16:40,607 INFO MainThread:2663926 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 22:16:40,676 INFO MainThread:2663926 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 22:16:40,677 INFO MainThread:2663926 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 22:16:41,012 INFO MainThread:2663926 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 22:16:41,012 INFO MainThread:2663926 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 22:16:41,012 INFO MainThread:2663926 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 22:16:41,012 INFO MainThread:2663926 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 22:16:41,014 INFO MainThread:2663926 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 22:16:41,014 INFO MainThread:2663926 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/tb, True +2025-01-21 22:16:44,562 INFO MainThread:2663926 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 22:21:57,515 WARNING MsgRouterThr:2663926 [router.py:message_loop():77] message_loop has been closed diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/code/FISH/eval_robot.py b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/config.yaml b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..851b2b83aa468c1f4298f267754460e5d2ea2ceb --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_1' +wandb_notes: + desc: null + value: 4295_1_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737515800 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/diff.patch b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..0d1d1c2d34392be0466354e2f5a3da5c22dfe3f1 --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/diff.patch @@ -0,0 +1,24 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..3a697a3 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False +@@ -144,8 +144,8 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '3465_2' # seed 2 + + # crp_D+mask+contact+act history +-wandb_run_id: '4295_0' +-# wandb_run_id: '4295_1' # seed 1 ++# wandb_run_id: '4295_0' ++wandb_run_id: '4295_1' # seed 1 + # wandb_run_id: '4295_2' # seed 2 + + # RGBD+mask+act history diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json new file mode 100644 index 0000000000000000000000000000000000000000..db88f8dce313fe228bb6c4839d50146f71aa6144 --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[20, 20, 1.0, 0.9946215747797684, 1, 0.8453640490079612, 1, 0.865594230519132, 1, 1, 1]]} \ No newline at end of file diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4d54d67fe89e5d748be742eede66b2a584557abf --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54e7fe314d9c986a5f0c339b114e70967262f53507592df0bdb70c184643fc31 +size 1473054 diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e3bbac8eaff7ab815f69b4b2d5c021614b954aad --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eac294d6ae1c852d4b2d5e31891d40e9a13ff5209555f8040faf7101096d7db9 +size 1479950 diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9f8c1441af0b2f25db351e02210481bdc3544f7 --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1e3df16fecb0ef689e4dde6e759b4639cd5eb141e92f708d145e9427a95270e +size 1471318 diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64511be74df573103663ea7db4d7cc6efbea28a8 --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8f87ed816b532325ab6470a29da11664f2bdb55b46547c19c28ce3879ae3f16 +size 1518514 diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..daaf7639aeb8dc15d5fc2ed3735c60e15590ed9f --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63c7fb6457e58c00fb2ccb5dff55bc85027fcb297dd43bbc0c0dcca2ba00d203 +size 1500970 diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..7817f9197134c95d6ef6fef7c737d6ea2463ae23 --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log @@ -0,0 +1,276 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737515804.563466]: resetting environment +[INFO] [1737515804.572161]: cleared current plan +[INFO] [1737515804.573069]: moving to home +[INFO] [1737515805.575653]: reached home +[INFO] [1737515805.577005]: reset action history +[INFO] [1737515808.290269]: environment reset +Starting episode 0 +[INFO] [1737515808.294248]: resetting environment +[INFO] [1737515808.299969]: cleared current plan +[INFO] [1737515808.300777]: moving to home +[INFO] [1737515809.302175]: reached home +[INFO] [1737515809.303349]: reset action history +[INFO] [1737515812.012249]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737515819.394829]: resetting environment +[INFO] [1737515819.398813]: cleared current plan +[INFO] [1737515819.399722]: moving to home +[INFO] [1737515820.402655]: reached home +[INFO] [1737515820.403921]: reset action history +[INFO] [1737515823.111838]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/episode_rosbags/episode_0_2025-01-21-22-17-03.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2663926.log +started roslaunch server http://158.130.50.37:36461/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2664232] +process[print_text-2]: started with pid [2664256] +process[rosbag_record-3]: started with pid [2664257] +started bagging! +For topic gripper_width: timestamp difference is -111048164 for nearest: 1737515828112505017 - target: 1737515828223553181 at idx 61 +[INFO] [1737515858.012015]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737515858.760231]: Stored episode 1. +[INFO] [1737515858.760570]: Saving video... +[INFO] [1737515858.954407]: Video saved! +Starting episode 1 +[INFO] [1737515858.975490]: resetting environment +[INFO] [1737515858.986403]: cleared current plan +[INFO] [1737515858.988625]: moving to home +[INFO] [1737515862.591999]: reached home +[INFO] [1737515862.592767]: reset action history +[INFO] [1737515865.304298]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737515868.568650]: resetting environment +[INFO] [1737515868.577390]: cleared current plan +[INFO] [1737515868.577583]: moving to home +[INFO] [1737515869.579024]: reached home +[INFO] [1737515869.579451]: reset action history +[INFO] [1737515872.287739]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/episode_rosbags/episode_1_2025-01-21-22-17-52.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2663926.log +started roslaunch server http://158.130.50.37:39399/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2664380] +process[print_text-5]: started with pid [2664381] +process[rosbag_record-6]: started with pid [2664382] +started bagging! +For topic gripper_width: timestamp difference is -103556087 for nearest: 1737515906312460969 - target: 1737515906416017056 at idx 249 +[INFO] [1737515907.008736]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1737515907.727868]: Stored episode 2. +[INFO] [1737515907.729141]: Saving video... +[INFO] [1737515908.051791]: Video saved! +Starting episode 2 +[INFO] [1737515908.083414]: resetting environment +[INFO] [1737515908.110928]: cleared current plan +[INFO] [1737515908.117749]: moving to home +[INFO] [1737515911.727594]: reached home +[INFO] [1737515911.729158]: reset action history +[INFO] [1737515914.499918]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737515926.063835]: resetting environment +[INFO] [1737515926.073058]: cleared current plan +[INFO] [1737515926.073280]: moving to home +[INFO] [1737515927.074723]: reached home +[INFO] [1737515927.075164]: reset action history +[INFO] [1737515929.785419]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/episode_rosbags/episode_2_2025-01-21-22-18-49.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2663926.log +started roslaunch server http://158.130.50.37:38415/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2664547] +process[print_text-8]: started with pid [2664548] +process[rosbag_record-9]: started with pid [2664549] +started bagging! +For topic gripper_width: timestamp difference is 118355379 for nearest: 1737515933579243050 - target: 1737515933460887671 at idx 49 +[INFO] [1737515964.773480]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1737515965.578905]: Stored episode 3. +[INFO] [1737515965.582170]: Saving video... +[INFO] [1737515965.880848]: Video saved! +Starting episode 3 +[INFO] [1737515965.926708]: resetting environment +[INFO] [1737515965.964108]: cleared current plan +[INFO] [1737515965.965257]: moving to home +[INFO] [1737515969.571209]: reached home +[INFO] [1737515969.573143]: reset action history +[INFO] [1737515972.346348]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737515974.354860]: resetting environment +[INFO] [1737515974.370023]: cleared current plan +[INFO] [1737515974.370709]: moving to home +[INFO] [1737515975.373550]: reached home +[INFO] [1737515975.375926]: reset action history +[INFO] [1737515978.175937]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/episode_rosbags/episode_3_2025-01-21-22-19-38.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2663926.log +started roslaunch server http://158.130.50.37:40553/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2664690] +process[print_text-11]: started with pid [2664691] +process[rosbag_record-12]: started with pid [2664692] +started bagging! +For topic gripper_width: timestamp difference is 103452644 for nearest: 1737515997779149494 - target: 1737515997675696850 at idx 249 +For topic gripper_width: timestamp difference is 108225673 for nearest: 1737516003679129689 - target: 1737516003570904016 at idx 249 +[INFO] [1737516013.971975]: Storing episode... +[rosbag_record-12] killing on exit[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1737516014.762187]: Stored episode 4. +[INFO] [1737516014.767680]: Saving video... +[INFO] [1737516015.143556]: Video saved! +Starting episode 4 +[INFO] [1737516015.198130]: resetting environment +[INFO] [1737516015.233973]: cleared current plan +[INFO] [1737516015.234200]: moving to home +[INFO] [1737516018.846559]: reached home +[INFO] [1737516018.863288]: reset action history +[INFO] [1737516021.668230]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737516029.074162]: resetting environment +[INFO] [1737516029.105341]: cleared current plan +[INFO] [1737516029.107538]: moving to home +[INFO] [1737516030.120948]: reached home +[INFO] [1737516030.124428]: reset action history +[INFO] [1737516032.913495]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/episode_rosbags/episode_4_2025-01-21-22-20-32.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2663926.log +started roslaunch server http://158.130.50.37:43529/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2664829] +process[print_text-14]: started with pid [2664830] +process[rosbag_record-15]: started with pid [2664831] +started bagging! +For topic gripper_width: timestamp difference is 110446076 for nearest: 1737516047679123740 - target: 1737516047568677664 at idx 207 +For topic gripper_width: timestamp difference is 106045775 for nearest: 1737516055179125123 - target: 1737516055073079348 at idx 249 +[INFO] [1737516067.966798]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1737516068.821710]: Stored episode 5. +[INFO] [1737516068.827544]: Saving video... +[WARN] [1737516069.286362]: Plan exhausted +[WARN] [1737516069.342318]: Plan exhausted +[WARN] [1737516069.371039]: Plan exhausted +[WARN] [1737516069.403898]: Plan exhausted +[WARN] [1737516069.446231]: Plan exhausted +[WARN] [1737516069.488185]: Plan exhausted +[WARN] [1737516069.524571]: Plan exhausted +[WARN] [1737516069.566774]: Plan exhausted +[INFO] [1737516069.469204]: Video saved! +[INFO] [1737516069.546451]: resetting environment +[INFO] [1737516069.577017]: cleared current plan +[INFO] [1737516069.577234]: moving to home +[INFO] [1737516073.180457]: reached home +[INFO] [1737516073.183862]: reset action history +[INFO] [1737516076.050640]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/requirements.txt b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-metadata.json b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b80020dab0a7a26bb239dd017ff77fcac4455a50 --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T03:16:40.702062", + "startedAt": "2025-01-22T03:16:40.305946", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3565.2328749999997, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4502.166, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 465.6271781921387 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b7c308027a27c56d2b7e45b8547341a16c133a32 --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "54e7fe314d9c986a5f0c339b114e70967262f53507592df0bdb70c184643fc31", "size": 1473054, "path": "media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4"}, "global_step": 665, "_timestamp": 1737516085.954643, "_runtime": 285.64124512672424, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "eac294d6ae1c852d4b2d5e31891d40e9a13ff5209555f8040faf7101096d7db9", "size": 1479950, "path": "media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4"}, "eval/num_success": 5.0, "episode": 20.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "f1e3df16fecb0ef689e4dde6e759b4639cd5eb141e92f708d145e9427a95270e", "size": 1471318, "path": "media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "a8f87ed816b532325ab6470a29da11664f2bdb55b46547c19c28ce3879ae3f16", "size": 1518514, "path": "media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "63c7fb6457e58c00fb2ccb5dff55bc85027fcb297dd43bbc0c0dcca2ba00d203", "size": 1500970, "path": "media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://ytndtei6ce09pvczzwl8hj0abxmebk6dsl7iyk1481ynzsznn1gkzckn09640hayn8v34gz4mw4yiv3y0cc5oltpphiiy2keqev2bi0b3amjiakb4mafdt1cjfzxd51x/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://zhqazmwb47lh3okyi13uflxi9m6z6causxt01k7cx6u9j80d1f36fc14eop1twsvk18n04uoueg7mrrg3kkqzv6ouod9aggfegdoris2hnzzag7v2fnypcu6hw1gnay0:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 20, "total_num_episodes": 20, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9946215747797684, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.8453640490079612, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.865594230519132, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "c68d11fd8ad6b790b7a949bb003b7716c3f8e083e007b71762bd414593b8472d", "size": 419, "artifact_path": "wandb-client-artifact://y6hsrnhf2mkt41dgefqppkwqbtr7rqwvkgppxiagc8mcqgszkejhy15xr2hq8bt6k1xan1t0jyfpq58y88sy0nzgzirg6fgepqtxxngpq6r5f8vndp1l8pikgyhjmt4e/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://vqnrpngqf1sg8dy3ehqyh72vbrk1fvdimouoxlm6g9nfs9hegxo8u4pi5rxou5a8g9zxscg8o2zidqa4r5dr64aotwa75ovhoiefqx3uiw1a6bnugifg1duup6brz1mx:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 306}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9946215748786926, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.8453640341758728, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.865594208240509, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/logs/debug-internal.log b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a2f7df02fa6dabc0ac08ef292298ffc4b6cd6fa6 --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/logs/debug-internal.log @@ -0,0 +1,826 @@ +2025-01-21 22:16:40,313 INFO StreamThr :2664121 [internal.py:wandb_internal():85] W&B internal server running at pid: 2664121, started at: 2025-01-21 22:16:40.312824 +2025-01-21 22:16:40,314 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status +2025-01-21 22:16:40,324 INFO WriterThread:2664121 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/run-zxwyz3xz.wandb +2025-01-21 22:16:40,325 DEBUG SenderThread:2664121 [sender.py:send():379] send: header +2025-01-21 22:16:40,336 DEBUG SenderThread:2664121 [sender.py:send():379] send: run +2025-01-21 22:16:40,594 INFO SenderThread:2664121 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files +2025-01-21 22:16:40,594 INFO SenderThread:2664121 [sender.py:_start_run_threads():1188] run started: zxwyz3xz with start time 1737515800.313398 +2025-01-21 22:16:40,608 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 22:16:40,608 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: check_version +2025-01-21 22:16:40,682 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 22:16:40,686 DEBUG HandlerThread:2664121 [system_info.py:__init__():26] System info init +2025-01-21 22:16:40,686 DEBUG HandlerThread:2664121 [system_info.py:__init__():41] System info init done +2025-01-21 22:16:40,687 INFO HandlerThread:2664121 [system_monitor.py:start():194] Starting system monitor +2025-01-21 22:16:40,687 INFO SystemMonitor:2664121 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 22:16:40,687 INFO HandlerThread:2664121 [system_monitor.py:probe():214] Collecting system info +2025-01-21 22:16:40,687 INFO SystemMonitor:2664121 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 22:16:40,688 INFO SystemMonitor:2664121 [interfaces.py:start():188] Started disk monitoring +2025-01-21 22:16:40,689 INFO SystemMonitor:2664121 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 22:16:40,690 INFO SystemMonitor:2664121 [interfaces.py:start():188] Started memory monitoring +2025-01-21 22:16:40,691 INFO SystemMonitor:2664121 [interfaces.py:start():188] Started network monitoring +2025-01-21 22:16:40,701 DEBUG HandlerThread:2664121 [system_info.py:probe():152] Probing system +2025-01-21 22:16:40,703 DEBUG HandlerThread:2664121 [system_info.py:_probe_git():137] Probing git +2025-01-21 22:16:40,708 DEBUG HandlerThread:2664121 [system_info.py:_probe_git():145] Probing git done +2025-01-21 22:16:40,708 DEBUG HandlerThread:2664121 [system_info.py:probe():200] Probing system done +2025-01-21 22:16:40,708 DEBUG HandlerThread:2664121 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T03:16:40.702062', 'startedAt': '2025-01-22T03:16:40.305946', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3565.2328749999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4502.166, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 465.6271781921387}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 22:16:40,708 INFO HandlerThread:2664121 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 22:16:40,708 INFO HandlerThread:2664121 [system_monitor.py:probe():227] Publishing system info +2025-01-21 22:16:40,708 DEBUG HandlerThread:2664121 [system_info.py:_save_code():44] Saving code +2025-01-21 22:16:40,710 DEBUG HandlerThread:2664121 [system_info.py:_save_code():67] Saving code done +2025-01-21 22:16:40,710 DEBUG HandlerThread:2664121 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 22:16:40,772 DEBUG HandlerThread:2664121 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 22:16:40,772 INFO HandlerThread:2664121 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 22:16:40,776 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:16:40,776 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 22:16:40,776 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 22:16:40,776 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 22:16:41,012 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 22:16:41,012 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: python_packages +2025-01-21 22:16:41,012 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:16:41,012 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:16:41,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:41,015 INFO HandlerThread:2664121 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/tb" + save: true +} +control { + relay_id: "bb8cd6a3072c49c789c67331dd91f729" +} +_info { + stream_id: "zxwyz3xz" +} + +2025-01-21 22:16:41,015 DEBUG HandlerThread:2664121 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 22:16:41,049 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/4m36kohu-wandb-metadata.json +2025-01-21 22:16:41,050 INFO wandb-upload_1:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/p0gake1q-code/FISH/eval_robot.py +2025-01-21 22:16:41,066 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/6zjdpwkl-diff.patch +2025-01-21 22:16:41,110 DEBUG SenderThread:2664121 [sender.py:send():379] send: telemetry +2025-01-21 22:16:41,111 DEBUG SenderThread:2664121 [sender.py:send():379] send: tbrecord +2025-01-21 22:16:41,111 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:16:41,111 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 with policy live +2025-01-21 22:16:41,304 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/10cc6yr0-events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:16:41,595 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/diff.patch +2025-01-21 22:16:41,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-metadata.json +2025-01-21 22:16:41,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/code/FISH/eval_robot.py +2025-01-21 22:16:41,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/requirements.txt +2025-01-21 22:16:41,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:16:41,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:16:41,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/code +2025-01-21 22:16:41,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/code/FISH +2025-01-21 22:16:42,012 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:43,012 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:43,595 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:16:44,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:44,563 DEBUG SenderThread:2664121 [sender.py:send():379] send: config +2025-01-21 22:16:45,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:45,576 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:16:45,595 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:16:46,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:47,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:47,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:16:48,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:49,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:49,596 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:16:50,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:51,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:16:51,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:51,597 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:16:52,053 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:53,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:53,598 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:16:54,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:55,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:56,012 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:16:56,012 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:16:56,053 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:56,113 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:16:57,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:58,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:16:59,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:00,015 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:01,015 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:01,600 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:02,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:02,015 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:03,015 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:04,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:05,025 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:05,601 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:06,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:07,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:07,025 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:08,025 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:09,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:09,603 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:10,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:11,012 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:17:11,012 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:17:11,053 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:12,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:12,174 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:12,604 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/config.yaml +2025-01-21 22:17:13,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:14,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:15,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:16,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:17,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:18,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:18,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:19,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:20,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:21,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:22,028 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:23,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:23,028 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:24,028 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:25,028 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:26,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:17:26,013 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:17:26,057 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:27,028 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:28,028 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:28,184 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:29,029 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:30,029 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:31,029 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:32,029 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:33,029 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:34,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:34,030 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:35,030 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:36,030 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:37,030 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:38,053 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:38,973 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:17:38,973 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 with policy now +2025-01-21 22:17:38,974 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:17:38,974 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:17:38,975 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:17:38,976 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:17:39,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:39,258 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/iu2ellmg-media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 +2025-01-21 22:17:39,611 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:39,611 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:17:39,611 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 +2025-01-21 22:17:39,612 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval +2025-01-21 22:17:39,612 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos +2025-01-21 22:17:39,612 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media +2025-01-21 22:17:39,944 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/t8bfbyra-media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 +2025-01-21 22:17:39,989 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:40,031 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:40,691 DEBUG SystemMonitor:2664121 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 22:17:40,692 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:17:41,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:17:41,013 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:17:41,053 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:41,612 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:42,031 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:43,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:43,612 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:44,031 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:45,031 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:45,304 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:46,031 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:47,031 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:47,614 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:48,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:48,614 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:17:48,865 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/8lb3sby5-events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:17:49,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:49,614 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:50,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:50,621 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:51,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:51,615 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:52,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:53,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:53,616 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:17:54,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:55,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:56,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:17:56,013 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:17:56,057 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:56,082 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:17:57,033 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:58,033 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:17:59,033 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:00,033 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:01,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:02,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:02,033 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:03,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:04,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:05,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:06,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:07,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:07,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:08,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:09,034 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:10,035 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:10,700 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:18:11,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:18:11,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:18:11,057 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:12,035 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:12,170 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:13,035 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:14,035 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:15,035 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:16,035 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:17,036 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:18,015 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:18,036 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:19,036 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:20,038 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:21,037 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:22,037 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:23,015 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:23,037 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:24,037 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:25,037 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:26,013 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:18:26,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:18:26,061 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:27,038 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:27,627 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:28,039 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:28,054 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:28,101 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:18:28,101 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:18:28,102 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 with policy now +2025-01-21 22:18:28,102 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:18:28,102 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:18:28,124 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:18:28,366 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/tcov7fon-media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 +2025-01-21 22:18:28,627 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:18:28,627 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 +2025-01-21 22:18:28,628 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval +2025-01-21 22:18:28,943 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/ddsqehya-media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 +2025-01-21 22:18:29,046 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:29,628 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:30,045 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:31,060 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:32,040 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:33,043 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:33,630 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:33,773 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:34,054 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:35,042 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:35,631 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:36,043 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:37,043 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:38,043 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:39,017 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:39,045 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:40,043 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:40,693 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:18:41,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:18:41,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:18:41,057 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:42,044 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:43,044 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:44,045 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:44,091 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:45,046 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:46,044 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:46,435 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:18:46,435 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:18:46,436 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:18:46,635 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:18:46,635 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:18:47,044 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:47,635 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:48,044 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:49,044 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:49,117 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:49,637 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:50,144 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:51,123 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:51,637 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:52,124 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:53,124 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:54,124 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:54,625 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:18:55,124 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:55,639 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:18:56,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:18:56,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:18:56,124 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:57,125 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:58,125 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:18:59,125 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:00,125 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:00,130 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:01,125 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:02,125 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:03,125 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:04,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:05,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:06,020 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:06,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:07,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:08,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:09,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:10,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:10,693 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:19:11,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:19:11,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:19:11,105 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:11,128 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:12,127 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:13,126 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:14,127 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:15,127 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:16,127 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:17,020 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:17,127 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:18,127 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:19,127 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:20,128 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:21,128 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:22,022 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:22,128 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:23,128 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:24,128 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:25,128 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:25,650 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:25,925 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:19:25,925 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 with policy now +2025-01-21 22:19:25,925 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:19:25,926 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:19:25,926 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:19:25,941 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:19:26,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:19:26,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:19:26,131 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:26,176 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/9x8g5qjh-media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 +2025-01-21 22:19:26,650 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:19:26,651 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 +2025-01-21 22:19:26,651 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval +2025-01-21 22:19:26,957 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/w8aq67zh-media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 +2025-01-21 22:19:27,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:27,154 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:27,651 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:28,137 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:29,143 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:30,145 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:31,146 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:31,652 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:32,146 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:32,347 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:33,144 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:33,653 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:34,142 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:34,653 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:19:34,896 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/lrx0e4nt-events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:19:35,143 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:35,276 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:19:35,277 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:19:35,277 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:19:35,653 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:19:35,654 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:36,143 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:37,143 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:37,654 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:38,024 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:38,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:39,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:39,655 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:40,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:40,693 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:19:41,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:19:41,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:19:41,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:41,656 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:19:42,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:43,145 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:43,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:44,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:45,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:46,147 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:47,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:48,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:49,022 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:49,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:50,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:51,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:52,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:53,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:54,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:55,022 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:19:55,148 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:56,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:19:56,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:19:56,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:57,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:58,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:59,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:19:59,662 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:00,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:00,841 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:01,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:02,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:03,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:04,149 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:05,150 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:05,664 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:06,022 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:06,150 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:07,150 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:08,150 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:09,150 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:10,151 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:10,694 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:20:11,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:20:11,014 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:20:11,120 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:11,152 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:12,152 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:13,152 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:14,152 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:15,156 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:15,189 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:20:15,189 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 with policy now +2025-01-21 22:20:15,229 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:20:15,229 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:20:15,230 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:20:15,234 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:20:15,447 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/9f5ij4ht-media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 +2025-01-21 22:20:15,667 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:20:15,667 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:15,667 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 +2025-01-21 22:20:15,667 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval +2025-01-21 22:20:15,959 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/9qsuslpj-media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 +2025-01-21 22:20:16,156 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:17,025 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:17,163 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:17,668 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:18,177 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:19,164 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:19,668 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:20,164 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:21,172 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:21,669 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:22,158 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:22,713 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:23,168 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:23,670 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:24,168 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:25,161 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:26,019 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:20:26,020 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:20:26,162 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:27,166 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:28,104 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:28,164 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:29,161 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:29,344 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:20:29,344 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:20:29,345 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:20:29,672 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:20:29,672 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:20:30,169 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:31,170 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:31,673 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:32,172 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:33,172 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:33,195 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:34,170 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:35,170 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:35,674 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:36,170 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:37,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:38,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:39,026 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:39,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:40,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:40,694 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:20:41,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:20:41,015 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:20:41,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:42,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:43,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:44,171 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:44,184 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:45,172 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:46,172 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:47,172 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:48,175 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:49,175 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:49,679 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:49,725 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:50,175 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:51,175 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:52,175 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:53,175 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:54,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:55,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:20:55,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:56,014 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:20:56,015 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:20:56,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:57,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:57,681 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:20:58,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:20:59,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:00,137 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:00,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:01,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:02,176 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:03,177 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:04,177 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:05,177 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:06,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:06,177 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:07,177 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:08,180 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:09,193 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:09,542 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:21:09,542 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 with policy now +2025-01-21 22:21:09,545 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:21:09,545 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:21:09,545 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:09,581 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:09,756 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:21:09,767 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:09,767 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 +2025-01-21 22:21:09,767 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval +2025-01-21 22:21:09,794 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/bz2lyf2v-media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 +2025-01-21 22:21:10,080 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/d87scncr-media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 +2025-01-21 22:21:10,215 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:10,695 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:21:11,029 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:21:11,029 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:21:11,139 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:11,193 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:11,756 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:12,199 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:13,221 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:14,200 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:15,199 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:15,758 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:16,204 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:17,093 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:17,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:17,758 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:18,203 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:19,202 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:20,214 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:21,213 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:22,211 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:23,041 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:23,209 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:24,216 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:24,761 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:21:25,013 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/f59lbj9f-events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:21:25,223 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:25,412 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:21:25,413 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:25,413 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:25,433 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:21:25,433 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:21:25,668 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:21:25,761 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:21:25,761 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:25,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:21:25,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media +2025-01-21 22:21:25,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table +2025-01-21 22:21:25,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval +2025-01-21 22:21:25,952 INFO wandb-upload_0:2664121 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp35brkh83 +2025-01-21 22:21:25,956 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/vob1phb9-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:21:26,017 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:21:26,027 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:21:26,040 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:21:26,208 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:26,491 INFO SenderThread:2664121 [sender.py:send_request_log_artifact():1525] logged artifact run-zxwyz3xz-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzczMzAwNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwNzM1Ng==', 'latestArtifact': None}} +2025-01-21 22:21:26,492 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:21:26,492 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-21 22:21:26,492 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:21:26,492 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,492 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,493 DEBUG SenderThread:2664121 [sender.py:send():379] send: telemetry +2025-01-21 22:21:26,493 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,496 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,497 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,497 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,497 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,497 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,498 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,498 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,498 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,498 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,499 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,499 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,499 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,499 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,500 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,500 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,500 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,500 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,501 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,501 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,501 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:26,501 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:26,501 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:21:26,650 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:21:26,743 INFO wandb-upload_1:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/w97akgo4-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:21:26,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:21:26,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:21:26,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-21 22:21:26,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval +2025-01-21 22:21:26,992 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/5p5mfol4-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-21 22:21:27,009 INFO wandb-upload_1:2664121 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpj108ekry +2025-01-21 22:21:27,208 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:27,402 INFO SenderThread:2664121 [sender.py:send_request_log_artifact():1525] logged artifact run-zxwyz3xz-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzczMzAyOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwNzM1OQ==', 'latestArtifact': None}} +2025-01-21 22:21:27,402 DEBUG SenderThread:2664121 [sender.py:send():379] send: files +2025-01-21 22:21:27,403 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json with policy now +2025-01-21 22:21:27,403 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:21:27,403 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:27,403 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:27,696 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/nsx7eczf-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-21 22:21:27,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:21:27,762 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:28,208 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:28,404 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:29,208 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:30,208 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:31,208 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:32,208 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:33,209 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:34,035 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:34,209 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:34,765 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/config.yaml +2025-01-21 22:21:35,209 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:36,209 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:37,209 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:38,209 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:39,214 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:40,032 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:40,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:40,695 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:21:41,017 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:21:41,017 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:21:41,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:42,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:43,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:44,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:45,114 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:45,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:46,210 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:46,725 DEBUG SenderThread:2664121 [sender.py:send():379] send: exit +2025-01-21 22:21:46,725 INFO SenderThread:2664121 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 22:21:46,725 INFO SenderThread:2664121 [sender.py:send_exit():588] handling runtime: 306 +2025-01-21 22:21:46,726 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:46,726 INFO SenderThread:2664121 [sender.py:send_exit():594] send defer +2025-01-21 22:21:46,726 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:46,726 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 22:21:46,726 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:46,726 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 22:21:46,726 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 1 +2025-01-21 22:21:46,726 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:46,726 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 22:21:46,727 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:46,727 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 22:21:46,727 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 2 +2025-01-21 22:21:46,727 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:46,727 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 22:21:46,727 INFO HandlerThread:2664121 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 22:21:46,727 DEBUG SystemMonitor:2664121 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 22:21:46,727 INFO HandlerThread:2664121 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 22:21:46,727 DEBUG SystemMonitor:2664121 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 22:21:46,727 INFO HandlerThread:2664121 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 22:21:46,770 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:21:46,781 INFO HandlerThread:2664121 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 22:21:46,781 INFO HandlerThread:2664121 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 22:21:46,782 INFO HandlerThread:2664121 [interfaces.py:finish():200] Joined network monitor +2025-01-21 22:21:46,782 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:46,782 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 22:21:46,782 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 3 +2025-01-21 22:21:46,782 DEBUG SenderThread:2664121 [sender.py:send():379] send: stats +2025-01-21 22:21:46,782 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:46,782 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 22:21:46,783 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:46,783 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 22:21:46,783 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 4 +2025-01-21 22:21:46,783 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:46,783 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 22:21:47,771 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:53,418 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:21:53,418 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:53,418 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 22:21:53,419 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:53,419 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 22:21:53,419 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 5 +2025-01-21 22:21:53,420 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:21:53,420 DEBUG SenderThread:2664121 [sender.py:send():379] send: history +2025-01-21 22:21:53,420 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:21:53,420 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:53,420 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 22:21:53,421 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:53,421 DEBUG SenderThread:2664121 [sender.py:send():379] send: summary +2025-01-21 22:21:53,421 INFO SenderThread:2664121 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:21:53,422 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:53,422 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 22:21:53,422 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 6 +2025-01-21 22:21:53,422 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:53,422 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 22:21:53,422 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:53,422 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 22:21:53,422 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 7 +2025-01-21 22:21:53,422 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:21:53,422 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:53,422 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 22:21:53,422 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:53,423 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 22:21:53,727 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:21:53,773 INFO Thread-12 :2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:21:55,331 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 8 +2025-01-21 22:21:55,331 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:21:55,332 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:55,332 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 22:21:55,332 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:55,332 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 22:21:55,332 INFO SenderThread:2664121 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 22:21:55,335 INFO SenderThread:2664121 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 22:21:55,337 INFO SenderThread:2664121 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 22:21:55,354 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 9 +2025-01-21 22:21:55,354 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:55,354 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 22:21:55,354 DEBUG SenderThread:2664121 [sender.py:send():379] send: artifact +2025-01-21 22:21:55,578 INFO SenderThread:2664121 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3MzQyMw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 22:21:55,578 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:55,578 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 22:21:55,578 INFO SenderThread:2664121 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 22:21:55,727 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:21:55,774 INFO SenderThread:2664121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:55,774 INFO SenderThread:2664121 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/config.yaml config.yaml +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json wandb-summary.json +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/requirements.txt requirements.txt +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-metadata.json wandb-metadata.json +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log output.log +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/diff.patch diff.patch +2025-01-21 22:21:55,775 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 22:21:55,777 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 media/videos/eval/4_eval_4_63c7fb6457e58c00fb2c.mp4 +2025-01-21 22:21:55,780 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 media/videos/eval/2_eval_2_f1e3df16fecb0ef689e4.mp4 +2025-01-21 22:21:55,780 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 media/videos/eval/0_eval_0_54e7fe314d9c986a5f0c.mp4 +2025-01-21 22:21:55,780 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 media/videos/eval/1_eval_1_eac294d6ae1c852d4b2d.mp4 +2025-01-21 22:21:55,781 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 media/videos/eval/3_eval_3_a8f87ed816b532325ab6.mp4 +2025-01-21 22:21:55,783 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-21 22:21:55,784 INFO SenderThread:2664121 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-21 22:21:55,784 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 10 +2025-01-21 22:21:55,784 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:21:55,784 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:55,784 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 22:21:55,784 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:55,784 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 22:21:55,784 INFO SenderThread:2664121 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:21:55,971 INFO wandb-upload_0:2664121 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/wandb-summary.json +2025-01-21 22:21:56,004 INFO wandb-upload_4:2664121 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/output.log +2025-01-21 22:21:56,014 INFO wandb-upload_2:2664121 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/config.yaml +2025-01-21 22:21:56,016 INFO wandb-upload_1:2664121 [upload_job.py:push():130] Uploaded file /tmp/tmp6nizv6g4wandb/mat6ces8-events.out.tfevents.1737515801.leonmkim-ROG-Strix-G15CS-G15CS.2663926.0 +2025-01-21 22:21:56,028 INFO wandb-upload_3:2664121 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/files/requirements.txt +2025-01-21 22:21:56,228 INFO Thread-11 (_thread_body):2664121 [sender.py:transition_state():614] send defer: 11 +2025-01-21 22:21:56,228 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:56,228 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 22:21:56,229 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:56,229 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 22:21:56,229 INFO SenderThread:2664121 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 22:21:56,230 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 12 +2025-01-21 22:21:56,230 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:56,230 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 22:21:56,230 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:56,230 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 22:21:56,230 INFO SenderThread:2664121 [file_stream.py:finish():601] file stream finish called +2025-01-21 22:21:56,445 INFO SenderThread:2664121 [file_stream.py:finish():605] file stream finish is done +2025-01-21 22:21:56,445 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 13 +2025-01-21 22:21:56,445 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:56,445 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 22:21:56,446 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:56,446 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 22:21:56,446 INFO SenderThread:2664121 [sender.py:transition_state():614] send defer: 14 +2025-01-21 22:21:56,446 DEBUG SenderThread:2664121 [sender.py:send():379] send: final +2025-01-21 22:21:56,446 DEBUG SenderThread:2664121 [sender.py:send():379] send: footer +2025-01-21 22:21:56,446 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:21:56,446 INFO HandlerThread:2664121 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 22:21:56,446 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: defer +2025-01-21 22:21:56,446 INFO SenderThread:2664121 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 22:21:56,447 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:21:56,447 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 22:21:56,447 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:21:56,448 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: server_info +2025-01-21 22:21:56,448 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:21:56,449 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 22:21:56,449 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 22:21:56,450 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:21:56,513 DEBUG SenderThread:2664121 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:21:56,513 INFO MainThread:2664121 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 22:21:56,514 INFO MainThread:2664121 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 22:21:56,514 INFO MainThread:2664121 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 22:21:56,514 DEBUG HandlerThread:2664121 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 22:21:56,514 INFO HandlerThread:2664121 [handler.py:finish():882] shutting down handler +2025-01-21 22:21:57,450 INFO WriterThread:2664121 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/run-zxwyz3xz.wandb +2025-01-21 22:21:57,513 INFO SenderThread:2664121 [sender.py:finish():1615] shutting down sender +2025-01-21 22:21:57,513 INFO SenderThread:2664121 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:21:57,514 INFO SenderThread:2664121 [file_pusher.py:join():175] waiting for file pusher diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/logs/debug.log b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..03c270d4116113d9fcd0cbc9b2ac0832c34e6700 --- /dev/null +++ b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Configure stats pid to 2663926 +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/settings +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 22:16:40,307 INFO MainThread:2663926 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 22:16:40,308 INFO MainThread:2663926 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/logs/debug.log +2025-01-21 22:16:40,308 INFO MainThread:2663926 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/logs/debug-internal.log +2025-01-21 22:16:40,308 INFO MainThread:2663926 [wandb_init.py:init():569] calling init triggers +2025-01-21 22:16:40,308 INFO MainThread:2663926 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_1', 'wandb_notes': '4295_1_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz'} +2025-01-21 22:16:40,308 INFO MainThread:2663926 [wandb_init.py:init():619] starting backend +2025-01-21 22:16:40,308 INFO MainThread:2663926 [wandb_init.py:init():623] setting up manager +2025-01-21 22:16:40,312 INFO MainThread:2663926 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 22:16:40,313 INFO MainThread:2663926 [wandb_init.py:init():631] backend started and connected +2025-01-21 22:16:40,324 INFO MainThread:2663926 [wandb_init.py:init():720] updated telemetry +2025-01-21 22:16:40,333 INFO MainThread:2663926 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 22:16:40,607 INFO MainThread:2663926 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 22:16:40,676 INFO MainThread:2663926 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 22:16:40,677 INFO MainThread:2663926 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 22:16:41,012 INFO MainThread:2663926 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 22:16:41,012 INFO MainThread:2663926 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 22:16:41,012 INFO MainThread:2663926 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 22:16:41,012 INFO MainThread:2663926 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 22:16:41,014 INFO MainThread:2663926 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 22:16:41,014 INFO MainThread:2663926 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_1/zxwyz3xz/tb, True +2025-01-21 22:16:44,562 INFO MainThread:2663926 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 22:21:57,515 WARNING MsgRouterThr:2663926 [router.py:message_loop():77] message_loop has been closed diff --git a/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/run-zxwyz3xz.wandb b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/run-zxwyz3xz.wandb new file mode 100644 index 0000000000000000000000000000000000000000..648a842f24c74e1121b45384527a753b6353ae65 Binary files /dev/null and b/zxwyz3xz/wandb/run-20250121_221640-zxwyz3xz/run-zxwyz3xz.wandb differ